From ede4c8d9775f2bc5572b4b5e2c2b33b2c9eb760d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 10:57:19 +0000 Subject: [PATCH 0001/2295] added hive_compute_table_stats.py, impala_compute_table_stats.py --- hive_compute_table_stats.py | 172 ++++++++++++++++++++++++++++++++++ impala_compute_table_stats.py | 170 +++++++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100755 hive_compute_table_stats.py create mode 100755 impala_compute_table_stats.py diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py new file mode 100755 index 000000000..59f13d2d5 --- /dev/null +++ b/hive_compute_table_stats.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a HiveServer2 and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Hive tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + conn = connect_db(args, None) + + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('ANALYZE TABLE {db}.{table} PARTITION({key}={value}) COMPUTE STATISTICS'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py new file mode 100755 index 000000000..669959f49 --- /dev/null +++ b/impala_compute_table_stats.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a Impalad and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Impala 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Impala tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impalad host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impalad port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + conn = connect_db(args, None) + + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('COMPUTE INCREMENTAL STATS {db}.{table} PARTITION({key}={value})'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() From a57ad82821d24572d41b49bc08178b003dfc42d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:50:13 +0000 Subject: [PATCH 0002/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cea4c6fae..6b034a1ac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0 +Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce From 28635feb83f6a04671871b74800590acae3b8680 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:50:36 +0000 Subject: [PATCH 0003/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6b034a1ac..af1b3c13d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce +Subproject commit af1b3c13d4a3cdf0897284ab31e19ad5b85044f2 From 585c2b2584cbb0dd0df24b09ab0c7ecb4856a833 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:51:29 +0000 Subject: [PATCH 0004/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93cc037e0..3dbf482b0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976 +Subproject commit 3dbf482b086e301d454511111c884814ce84acd2 From dcf4c49a7b628086e9fcfc5f407f436e668fbab9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 14:49:57 +0000 Subject: [PATCH 0005/2295] updated hive_compute_table_stats.py --- hive_compute_table_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index 59f13d2d5..1dd33b885 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -165,6 +165,7 @@ def compute_table_stats(conn, args, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) with conn.cursor() as table_cursor: log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table)) From 4c3dfef1572dedabf07f4c19d8c108f7efc35fde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 14:50:05 +0000 Subject: [PATCH 0006/2295] updated impala_compute_table_stats.py --- impala_compute_table_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index 669959f49..f545224bc 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -163,6 +163,7 @@ def compute_table_stats(conn, args, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) with conn.cursor() as table_cursor: log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table)) From ba23262fcb2d6e82734ae2fd5cfb924f64c63db6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 17:13:22 +0000 Subject: [PATCH 0007/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index af1b3c13d..545423ba2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit af1b3c13d4a3cdf0897284ab31e19ad5b85044f2 +Subproject commit 545423ba200b85def8c854315f1cb1d8f119ab24 From b77d9b65aece0a2ff3e276990a9c379700327980 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 17:13:22 +0000 Subject: [PATCH 0008/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3dbf482b0..dc3c170cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3dbf482b086e301d454511111c884814ce84acd2 +Subproject commit dc3c170cf80d4a41fdc2797fcc5316247eda920c From 13d81bb707adc1208bdc7cfa44492819691dbbde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 10:06:52 +0000 Subject: [PATCH 0009/2295] added regex try/except --- hive_compute_table_stats.py | 12 ++++++++---- impala_compute_table_stats.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index 1dd33b885..0306c44ae 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -108,11 +108,15 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) + conn = connect_db(args, None) log.info('querying databases') with conn.cursor() as db_cursor: diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index f545224bc..ddfaeeba8 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -106,11 +106,15 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) + conn = connect_db(args, None) log.info('querying databases') with conn.cursor() as db_cursor: From 4710498a37f28baa447cddf36c3a6a6382126936 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 23:35:14 +0000 Subject: [PATCH 0010/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cea4c6fae..1b52566af 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0 +Subproject commit 1b52566af83b38e372cbf0176fd3420e6bd7ff4e From efcebf203403da280b78958a26d7dd064e2fb356 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 23:35:32 +0000 Subject: [PATCH 0011/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93cc037e0..912ea1b62 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976 +Subproject commit 912ea1b622e0056bc4f51572dcf795e32a5ad097 From a295f828dd0837d7f6ca086bdc92d2dbcfefd8a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:32:54 +0000 Subject: [PATCH 0012/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b52566af..aa71f5453 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b52566af83b38e372cbf0176fd3420e6bd7ff4e +Subproject commit aa71f5453337b4404fb2c2bd082f506d7fcb7bd3 From c9bf73567f398d2f8a6c8fbba2ddbf9a75bffe32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:32:54 +0000 Subject: [PATCH 0013/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 912ea1b62..42a3563d6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 912ea1b622e0056bc4f51572dcf795e32a5ad097 +Subproject commit 42a3563d6277348774d6839e282672824ae694f1 From a17615b59072a0a0fa04f230fa44266a7ee67427 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:42:03 +0000 Subject: [PATCH 0014/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index aa71f5453..d6221cd31 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit aa71f5453337b4404fb2c2bd082f506d7fcb7bd3 +Subproject commit d6221cd31b65db515b0b755db3742e5c874f7a6d From 68d037e90671313adc0f3fbdd5a7b830adaadea2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:42:03 +0000 Subject: [PATCH 0015/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 42a3563d6..f46398a33 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 42a3563d6277348774d6839e282672824ae694f1 +Subproject commit f46398a3372530c148e1fdc760b2a2659618aa3c From bf226d32d93dafd1c1d0d56de7129bce2b62e79c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:43:54 +0000 Subject: [PATCH 0016/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d6221cd31..c6a900361 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d6221cd31b65db515b0b755db3742e5c874f7a6d +Subproject commit c6a9003610ddd2321dc28c2c7d69f98eb43ba7c7 From 8ac30eab41190918f9f063b3065edba2f69edfe5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:43:55 +0000 Subject: [PATCH 0017/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f46398a33..cf0435d86 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f46398a3372530c148e1fdc760b2a2659618aa3c +Subproject commit cf0435d8687fc0951c91ed5cb410a84bce31aca0 From bc0aea723c233b2fae0005d3f06044db1aad92c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 1 Dec 2019 21:53:10 +0000 Subject: [PATCH 0018/2295] added exception to not take Original Amount foreign currency column --- crunch_accounting_csv_statement_converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 34069c12b..477198dbf 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class CrunchAccountingCsvStatementConverter(CLI): @@ -169,7 +169,8 @@ def detect_columns(self, csvreader): positions['date'] = position elif 'Merchant Name' in value: positions['desc'] = position - elif 'Amount' in value: + # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33 + elif 'Amount' in value and not 'Original' in value: positions['amount'] = position elif 'Balance' in value: balance_position = position From e5d5f70bd8a9b9b593710dea8f46851b2276cbf1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Dec 2019 16:25:10 +0000 Subject: [PATCH 0019/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 82b1ab320..3058e28dc 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -42,6 +42,15 @@ https://github.com/cloudera/impyla/issues/286 +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + """ from __future__ import absolute_import @@ -58,7 +67,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -104,11 +113,13 @@ def parse_args(): parser.add_argument('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV (recommended, default is double quote \'"\')') parser.add_argument('-E', '--escapechar', help='Escape char if needed') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + parser.add_argument('-v', '--verbose', action='count', help='Verbose mode') args = parser.parse_args() if args.verbose: log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) if 'impala' in sys.argv[0]: if args.krb5_service_name == 'hive': @@ -123,6 +134,10 @@ def connect_db(args, database): auth_mechanism = None if args.kerberos: auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', args.krb5_service_name) + if args.ssl is True: + log.debug('ssl enabled') log.info('connecting to %s:%s database %s', args.host, args.port, database) return connect( @@ -139,7 +154,7 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + conn = connect_db(args, 'default') quoting = csv.QUOTE_ALL if args.quotechar == '': From 035a49b6be15b0cc1412117bc9e6a00c1c40f928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:14:31 +0000 Subject: [PATCH 0020/2295] updated ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b951f3244..4310b15a6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,7 +1,3 @@ -Please be as specific as possible when raising an issue. - -- what were you expecting -- what was the result -- what was the output - if running a CLI program please include the full debug output when running with program by using the `--debug` or `-v -v -v` switches for most tools, or setting `export DEBUG=1` environment variable, which also works with the shell scripts. +Please be specific about your issue and include debug output from running after setting `export DEBUG=1` in your shell. You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. From bb9dc4bd0f94c2f1c9e186be2aafc26502b2083a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:15:22 +0000 Subject: [PATCH 0021/2295] updated ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 4310b15a6..5f04c0a58 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ -Please be specific about your issue and include debug output from running after setting `export DEBUG=1` in your shell. +Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. From 8497d24fb64bbe29df167a87f77af48485d06448 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:16:31 +0000 Subject: [PATCH 0022/2295] updated validate_yaml.py --- validate_yaml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validate_yaml.py b/validate_yaml.py index 2852a1905..285f75ef8 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -49,7 +49,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class YamlValidatorTool(CLI): @@ -99,7 +99,8 @@ def check_yaml(self, content): if not self.get_opt('print'): if self.verbose > 2: try: - yaml.safe_load(content) + # TODO: doesn't validate network-policy.yaml in templates containing multiple yaml docs + yaml.safe_load_all(content) except yaml.YAMLError as _: print(_) die(self.invalid_yaml_msg) From 124cfc421e7d2329894a0ae53b5e07cc3d512270 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:09:18 +0000 Subject: [PATCH 0023/2295] added aws_users_access_key_age.py --- aws_users_access_key_age.py | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 aws_users_access_key_age.py diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py new file mode 100755 index 000000000..1a65f1085 --- /dev/null +++ b/aws_users_access_key_age.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Lists all AWS IAM users keys along with their ages, optionally filtering any older than a given number of days + +Output format is: + + + +Status is usually Active + +See also aws_users_access_key_age.sh for a similar version in the adjacent DevOps Bash Tools repo +This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +import datetime +import boto3 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_float + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class AWSUsersAccessKeysAge(CLI): + + def __init__(self): + super(AWSUsersAccessKeysAge, self).__init__() + self.age = None + self.only_active_keys = False + + def add_options(self): + self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + + def process_args(self): + self.only_active_keys = self.get_opt('only_active') + self.age = self.get_opt('age') + if self.age: + validate_float(self.age, 'age') + self.age = float(self.age) + + def run(self): + iam = boto3.client('iam') + user_paginator = iam.get_paginator('list_users') + now = datetime.datetime.utcnow() + age = 0 + if self.age: + age = self.age * 86400 + for users_response in user_paginator.paginate(): + for user_item in users_response['Users']: + username = user_item['UserName'] + key_paginator = iam.get_paginator('list_access_keys') + for keys_response in key_paginator.paginate(UserName=username): + self.process_key(keys_response, username, now, age) + log.info('Completed') + + def process_key(self, keys_response, username, now, age): + assert not keys_response['IsTruncated'] + for access_key_item in keys_response['AccessKeyMetadata']: + assert username == access_key_item['UserName'] + status = access_key_item['Status'] + if self.only_active_keys and not status: + continue + create_date = access_key_item['CreateDate'] + #if age: + # already cast to datetime.datetime + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + #if (now - create_date).total_seconds() < age: + # continue + print('{user:20}\t{status}\t{date}'.format( + user=username, + status=status, + date=create_date)) + + +if __name__ == '__main__': + AWSUsersAccessKeysAge().main() From 17fc67218a7fcba43d75a6c1f91aac28e02eb0fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:09:45 +0000 Subject: [PATCH 0024/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 1a65f1085..4705025d8 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -62,11 +62,11 @@ def __init__(self): def add_options(self): self.add_opt('--age', help='Return keys older than this N days') - self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + #self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') - self.age = self.get_opt('age') + #self.age = self.get_opt('age') if self.age: validate_float(self.age, 'age') self.age = float(self.age) From 17567fafe31399399060fe6e1a46d381ef476482 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:10:13 +0000 Subject: [PATCH 0025/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 4705025d8..41517a60d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -61,8 +61,8 @@ def __init__(self): self.only_active_keys = False def add_options(self): - self.add_opt('--age', help='Return keys older than this N days') - #self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + #self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') From f7bd965d0ad1bd74f4c3c8aad74b83c25bc417b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:00:47 +0000 Subject: [PATCH 0026/2295] added --age support to report only keys older than N days --- aws_users_access_key_age.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 41517a60d..9ccfd543d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -51,42 +51,41 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class AWSUsersAccessKeysAge(CLI): def __init__(self): super(AWSUsersAccessKeysAge, self).__init__() self.age = None + self.now = None self.only_active_keys = False def add_options(self): - #self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--age', help='Return keys older than N days') self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') - #self.age = self.get_opt('age') + self.age = self.get_opt('age') if self.age: validate_float(self.age, 'age') self.age = float(self.age) + self.age = self.age * 86400 def run(self): iam = boto3.client('iam') user_paginator = iam.get_paginator('list_users') - now = datetime.datetime.utcnow() - age = 0 - if self.age: - age = self.age * 86400 + self.now = datetime.datetime.utcnow() for users_response in user_paginator.paginate(): for user_item in users_response['Users']: username = user_item['UserName'] key_paginator = iam.get_paginator('list_access_keys') for keys_response in key_paginator.paginate(UserName=username): - self.process_key(keys_response, username, now, age) + self.process_key(keys_response, username) log.info('Completed') - def process_key(self, keys_response, username, now, age): + def process_key(self, keys_response, username): assert not keys_response['IsTruncated'] for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] @@ -94,11 +93,14 @@ def process_key(self, keys_response, username, now, age): if self.only_active_keys and not status: continue create_date = access_key_item['CreateDate'] - #if age: - # already cast to datetime.datetime + if self.age: + # already cast to datetime.datetime with tzinfo #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') - #if (now - create_date).total_seconds() < age: - # continue + # removing tzinfo for comparison to avoid below error + # - both are UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: + continue print('{user:20}\t{status}\t{date}'.format( user=username, status=status, From 18ef21ff3fe6b339bc5adefaa5a725394e4660f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:00:54 +0000 Subject: [PATCH 0027/2295] updated requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index bd5bdaa1c..77613f9cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ avro==1.8.1 #avro-python3==1.9.0 awscli==1.16.241 #bitarray==0.8.1 +#boto==2.49.0 +boto3==1.10.37 #cassandra-driver==3.6.0 dicttoxml==1.7.4 # Elasticsearch library must match major version :( From 08ab4ccce4b080e021a71320da3ebf061389681a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:41:12 +0000 Subject: [PATCH 0028/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 9ccfd543d..7523e7947 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -86,7 +86,7 @@ def run(self): log.info('Completed') def process_key(self, keys_response, username): - assert not keys_response['IsTruncated'] + #assert not keys_response['IsTruncated'] for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] status = access_key_item['Status'] From 855332eac4f957dd1509939aa03be1efb3825ca0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:42:40 +0000 Subject: [PATCH 0029/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 7523e7947..8116e21e6 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -34,9 +34,9 @@ from __future__ import print_function from __future__ import unicode_literals +import datetime import os import sys -import datetime import boto3 libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) From e42fe1ec4e1b7892fae8166b372478b5873c27a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:58:11 +0000 Subject: [PATCH 0030/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 8116e21e6..b107fdcb3 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -90,7 +90,7 @@ def process_key(self, keys_response, username): for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] status = access_key_item['Status'] - if self.only_active_keys and not status: + if self.only_active_keys and status != 'Active': continue create_date = access_key_item['CreateDate'] if self.age: From 4c0aa16586980d9dea90e3f9cc9ac8e2c0e049e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 13:55:15 +0000 Subject: [PATCH 0031/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index b107fdcb3..aab16bd38 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -62,8 +62,8 @@ def __init__(self): self.only_active_keys = False def add_options(self): - self.add_opt('--age', help='Return keys older than N days') - self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + self.add_opt('-a', '--age', help='Return keys older than N days') + self.add_opt('-o', '--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') From a0887a072afcac0128c3df64a5876ad17950b0d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 17:07:53 +0000 Subject: [PATCH 0032/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index aab16bd38..da6ca7509 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,9 +24,16 @@ Status is usually Active -See also aws_users_access_key_age.sh for a similar version in the adjacent DevOps Bash Tools repo +See also: + +aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo (https://github.com/harisekhon/devops-bash-tools) This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies +Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) + +check_aws_access_keys_age.py +check_aws_access_keys_disabled.py + """ from __future__ import absolute_import From 64dc96d6b065de477aae022ecf87325df1b12d8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Dec 2019 10:05:43 +0000 Subject: [PATCH 0033/2295] set timeout higher --- aws_users_access_key_age.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index da6ca7509..eedbcad26 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -26,7 +26,9 @@ See also: -aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo (https://github.com/harisekhon/devops-bash-tools) +aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo +- https://github.com/harisekhon/devops-bash-tools + This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) @@ -67,6 +69,7 @@ def __init__(self): self.age = None self.now = None self.only_active_keys = False + self.timeout_default = 300 def add_options(self): self.add_opt('-a', '--age', help='Return keys older than N days') From 7e549bbff0e9d15ee5acc7b86b40d2aaca6646b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Dec 2019 11:47:37 +0000 Subject: [PATCH 0034/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index eedbcad26..72b13fa7f 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,17 +24,21 @@ Status is usually Active -See also: +Uses Boto, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html -aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo -- https://github.com/harisekhon/devops-bash-tools +See also: This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo + - https://github.com/harisekhon/devops-bash-tools + Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) -check_aws_access_keys_age.py -check_aws_access_keys_disabled.py + check_aws_access_keys_age.py + check_aws_access_keys_disabled.py """ From ddffcc39ebbc9d2d421d119f2ace11e43c95313f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:06:17 +0000 Subject: [PATCH 0035/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 72b13fa7f..0d11e6616 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -22,7 +22,7 @@ -Status is usually Active +Status is Active or Inactive Uses Boto, read here for the list of ways to configure your AWS credentials: @@ -115,7 +115,7 @@ def process_key(self, keys_response, username): # TypeError: can't subtract offset-naive and offset-aware datetimes if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: continue - print('{user:20}\t{status}\t{date}'.format( + print('{user:20}\t{status:8}\t{date}'.format( user=username, status=status, date=create_date)) From d4eff1c534234f9f60d1514c0bf4b76ed87194b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:41:20 +0000 Subject: [PATCH 0036/2295] added support for AWS user/group arns --- anonymize.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 74cb19226..7a069e41f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.5' +__version__ = '0.9.6' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -297,6 +297,8 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), + 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(\ + group='({}/)*{}'.format(user_regex, user_regex)), 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -307,6 +309,8 @@ def __init__(self): 'user6': r'(?/){user}@'.format(user=user_regex), 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), + #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex), + 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(user='({}/)*{}'.format(user_regex, user_regex)), 'password': r'([-\.]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -396,10 +400,12 @@ def __init__(self): 'user5': 'for user ', 'user6': '@', 'user7': r'\1', + 'user8': r'\1\2', 'group': r'\1', 'group2': r'\1', 'group3': r'for group ', 'group4': r'\1', + 'group5': r'\1\2', 'password': r'\1', 'password2': r'\1:', 'password3': r'\1\2', @@ -792,16 +798,16 @@ def anonymize(self, line): line = self.anonymize_dynamic(_, line) if line is None: if method: - raise AssertionError('anonymize_{} returned None', _) + raise AssertionError('anonymize_{} returned None'.format(_)) else: - raise AssertionError('anonymize_dynamic({}, line)', _) + raise AssertionError('anonymize_dynamic({}, line)'.format(_)) line += line_ending return line def anonymize_dynamic(self, name, line): #log.debug('anonymize_dynamic(%s, %s)', name, line) if not isStr(line): - raise AssertionError('anonymize_dynamic: passed in non-string line: %s', line) + raise AssertionError('anonymize_dynamic: passed in non-string line: {}'.format(line)) line = self.dynamic_replace(name, line) for i in range(2, 101): name2 = '{}{}'.format(name, i) From f39382c662a5c2239f22bc0cf49a7665953ab674 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:41:24 +0000 Subject: [PATCH 0037/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 20ff7fa24..f30e9ac0a 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -430,6 +430,18 @@ dest[111]="127.0.0.1" src[112]="travis token: Abc123" dest[112]="travis token: " +src[113]="arn:aws:iam::123456789012:user/hari" +dest[113]="arn:aws:iam:::user/" + +src[114]="arn:aws:iam::123456789012:group/hari" +dest[114]="arn:aws:iam:::group/" + +src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" +dest[115]="arn:aws:iam:::user//*" + +src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[116]="arn:aws:iam:::group//*" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 7842b013daf8c8a490c4df6a36caaeca07d9e525 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:23:31 +0000 Subject: [PATCH 0038/2295] added --aws support --- anonymize.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 7a069e41f..aae935b51 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.6' +__version__ = '0.9.7' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -114,6 +114,7 @@ def __init__(self): self.hash_salt = None # order of iteration of application matters because we must do more specific matches before less specific ones self.anonymizations = OrderedDict([ + ('aws', False), ('ip_prefix', False), ('ip', False), ('subnet_mask', False), @@ -282,6 +283,14 @@ def __init__(self): # openssl uses -passin switch pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' self.regex = { + # arn:partition:service:region:account-id:resource-id + # arn:partition:service:region:account-id:resource-type/resource-id + # arn:partition:service:region:account-id:resource-type:resource-id + # eg. arn:aws:iam::123456789012:group/Development/product_1234/* + 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/-]+', + # arn:aws:s3:::my_corporate_bucket/Development/* + #'aws2': r'\b(arn:aws:s3:::)[^/]+', + 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/-]+', # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -384,6 +393,11 @@ def __init__(self): ldap_lambda_lowercase = lambda m: r'{}<{}>'.format(m.group(1), m.group(2).lower()) # will auto-infer replacements to not have to be explicit, use this only for override mappings self.replacements = { + # arn:partition:service:region:account-id:resource-id + # arn:partition:service:region:account-id:resource-type/resource-id + # arn:partition:service:region:account-id:resource-type:resource-id + 'aws': r'\1\2<\3>', + 'aws2': r'\1:', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -462,6 +476,7 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') + self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations'), self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ @@ -573,6 +588,7 @@ def process_options(self): if _ in ('subnet_mask', 'mac', 'group'): continue self.anonymizations[_] = self.get_opt(_) + log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_])) self._process_options_host() self._process_options_network() self._process_options_exceptions() From 2fb81ce44b6801685514ad55c2cebd4868eee434 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:23:38 +0000 Subject: [PATCH 0039/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index f30e9ac0a..fb0b8036a 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -437,10 +437,16 @@ src[114]="arn:aws:iam::123456789012:group/hari" dest[114]="arn:aws:iam:::group/" src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" -dest[115]="arn:aws:iam:::user//*" +dest[115]="arn:aws:iam:::user/*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group//*" +dest[116]="arn:aws:iam:::group/*" + +src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[116]="arn:aws:iam:::group/*" + +src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" +dest[117]="arn:aws:s3:::*" # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 222ec6e61cb63277ecdafb7fc3b2589bc7a72c3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:41:45 +0000 Subject: [PATCH 0040/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 27cd69ba9..760534d27 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Environment variables are supported for convenience and also to hide credentials - hostnames / domains / FQDNs - email addresses - IP + MAC addresses + - AWS ARNs - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings From 323e9c753fed21527c754bfe2643a23e21f3ca85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 09:48:57 +0000 Subject: [PATCH 0041/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 0d11e6616..a4b2e503d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -28,11 +28,11 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html -See also: - This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies - aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo +See also: + + aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering - https://github.com/harisekhon/devops-bash-tools Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) From 15b96613977395f020e389e330d9b95819404e07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 11:24:59 +0000 Subject: [PATCH 0042/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index a4b2e503d..89dbbfa02 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,7 +24,7 @@ Status is Active or Inactive -Uses Boto, read here for the list of ways to configure your AWS credentials: +Uses the Boto library, read here for the list of ways to configure your AWS credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html From 7c29886514bf91d4faa89b972566f3afa59b721d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 12:08:54 +0000 Subject: [PATCH 0043/2295] added aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 aws_users_pw_last_used.py diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py new file mode 100755 index 000000000..645121510 --- /dev/null +++ b/aws_users_pw_last_used.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Lists all AWS IAM users keys along with the dates and days since their passwords were last used, optionally filtering +for users whose passwords haven't been used in > N days + +Output format is: + + + +Uses Boto, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + +See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools + +https://github.com/harisekhon/devops-bash-tools + +https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import datetime +import os +import sys +from math import floor +import boto3 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_float + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class AWSUsersPasswordLastUsed(CLI): + + def __init__(self): + super(AWSUsersPasswordLastUsed, self).__init__() + self.age = None + self.now = None + self.timeout_default = 300 + + def add_options(self): + self.add_opt('-a', '--age', help='Return users with passwords last used more than N days ago') + + def process_args(self): + self.age = self.get_opt('age') + if self.age: + validate_float(self.age, 'age') + self.age = float(self.age) + + def run(self): + iam = boto3.client('iam') + user_paginator = iam.get_paginator('list_users') + self.now = datetime.datetime.utcnow() + for users_response in user_paginator.paginate(): + for user_item in users_response['Users']: + log.debug(user_item) + self.process_password_age(user_item) + log.info('Completed') + + def process_password_age(self, user_item): + # already cast to datetime.datetime with tzinfo + user = user_item['UserName'] + if 'PasswordLastUsed' in user_item: + password_last_used = user_item['PasswordLastUsed'] + # removing tzinfo for comparison to avoid below error + # - both are UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + datetime_delta = self.now - password_last_used.replace(tzinfo=None) + days = int(floor(datetime_delta.total_seconds() / 86400)) + if self.age and days <= self.age: + return + else: + password_last_used = 'N/A' + days = 'N/A' + print('{user:20s}\t{password_last_used}\t({days:>3} days)'.format( + user=user, + password_last_used=password_last_used, + days=days)) + + +if __name__ == '__main__': + AWSUsersPasswordLastUsed().main() From 761e86ddffa7f64515899925a1e4d8fd1f892f66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 12:10:20 +0000 Subject: [PATCH 0044/2295] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index 645121510..ae3557778 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -104,9 +104,9 @@ def process_password_age(self, user_item): else: password_last_used = 'N/A' days = 'N/A' - print('{user:20s}\t{password_last_used}\t({days:>3} days)'.format( + print('{user:20s}\t{password_last_used:25s}\t({days:>3} days)'.format( user=user, - password_last_used=password_last_used, + password_last_used=str(password_last_used), # without str() format string breaks with :25 days=days)) From 95c7e70ed16b9a82ecdd6cbf83fdb0e8436ea8ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:01:26 +0000 Subject: [PATCH 0045/2295] added json_to_yaml.py --- json_to_yaml.py | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 json_to_yaml.py diff --git a/json_to_yaml.py b/json_to_yaml.py new file mode 100755 index 000000000..32f36e22e --- /dev/null +++ b/json_to_yaml.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to convert JSON to YAML + +Reads any given files as JSON and prints the equivalent YAML to stdout for piping or redirecting to a file. + +Directories if given are detected and recursed, processing all files in the directory tree ending in a .json suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import json +import os +import re +import sys +import yaml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, ERRORS, log, log_option + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class JsonToYaml(CLI): + + def __init__(self): + # Python 2.x + super(JsonToYaml, self).__init__() + # Python 3.x + # super().__init__() + self.re_json_suffix = re.compile(r'.*\.json$', re.I) + + @staticmethod + def json_to_yaml(content, filepath=None): + try: + _ = json.loads(content) + except (KeyError, ValueError) as _: + file_detail = '' + if filepath is not None: + file_detail = ' in file \'{0}\''.format(filepath) + die("Failed to parse JSON{0}: {1}".format(file_detail, _)) + return yaml.safe_dump(_) + + def run(self): + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'%s' not found" % arg) + sys.exit(ERRORS['WARNING']) + if os.path.isfile(arg): + log_option('file', arg) + elif os.path.isdir(arg): + log_option('directory', arg) + else: + die("path '%s' could not be determined as either a file or directory" % arg) + for arg in self.args: + self.process_path(arg) + + def process_path(self, path): + if path == '-' or os.path.isfile(path): + self.process_file(path) + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for filename in files: + filepath = os.path.join(root, filename) + if self.re_json_suffix.match(filepath): + self.process_file(filepath) + else: + die("failed to determine if path '%s' is a file or directory" % path) + + def process_file(self, filepath): + log.debug('processing filepath \'%s\'', filepath) + if filepath == '-': + filepath = '' + if filepath == '': + self.json_to_yaml(sys.stdin.read()) + else: + with open(filepath) as _: + content = _.read() + print('---') + print(self.json_to_yaml(content, filepath=filepath)) + + +if __name__ == '__main__': + JsonToYaml().main() From 97396a6f9c998a254cf7961dc893ca299d488e44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:03:57 +0000 Subject: [PATCH 0046/2295] updated json_to_yaml.py --- json_to_yaml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/json_to_yaml.py b/json_to_yaml.py index 32f36e22e..078b2fbac 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -25,6 +25,10 @@ Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from standard input. +See also: + + https://www.commandlinefu.com/commands/view/12221/convert-json-to-yaml + """ from __future__ import absolute_import From b52aaa7aec8e830fa172f9915c9226fb87b31551 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:10:07 +0000 Subject: [PATCH 0047/2295] added tests/test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 tests/test_json_to_yaml.sh diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh new file mode 100755 index 000000000..4e4d2cae2 --- /dev/null +++ b/tests/test_json_to_yaml.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "$0")" && pwd)" + +cd "$srcdir"; + +. utils.sh +. ../bash-tools/lib/utils.sh + +section "JSON => YAML" + +cd .. + +tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" +#echo "tmpfile is $tmpfile" + +trap "rm -f $tmpfile" $TRAP_SIGNALS + +for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do + echo "running json_to_yaml.py $x" + ./json_to_yaml.py "$x" > "$tmpfile" + echo "now validating generated yaml" + ./validate_yaml.py "$tmpfile" + echo +done + +echo "recursing directory to convert all json to yaml > /dev/null" +./json_to_yaml.py cloudformation/ >/dev/null +echo "Success" From dbd24050ddfee24c4863fd4ad856d275c2f4a898 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:10:21 +0000 Subject: [PATCH 0048/2295] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 81a8f2669..095ec85f1 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -29,10 +29,10 @@ tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" trap "rm -f $tmpfile" $TRAP_SIGNALS -echo "running json_to_xml.py": +echo "running json_to_xml.py:" ../json_to_xml.py data/test.json | tee "$tmpfile" echo -echo "now validating generated xml": +echo "now validating generated xml:" ../validate_xml.py "$tmpfile" echo From e5840fbe4a8c21cfa4c57f09f57d2c02c4a25707 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:12:36 +0000 Subject: [PATCH 0049/2295] updated test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh index 4e4d2cae2..47c39af6e 100755 --- a/tests/test_json_to_yaml.sh +++ b/tests/test_json_to_yaml.sh @@ -39,6 +39,9 @@ for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_dou echo done -echo "recursing directory to convert all json to yaml > /dev/null" -./json_to_yaml.py cloudformation/ >/dev/null +echo "recursing directory to convert all json files under a directory tree to yaml" +./json_to_yaml.py cloudformation/ > "$tmpfile" +# TODO: fix validate_yaml.py to work on multi-yamls with --- and re-enable +#echo "now validating generated yaml" +#./validate_yaml.py "$tmpfile" echo "Success" From 4bdcbd0027d71811c13a909ede0cad66a60974c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:40:25 +0000 Subject: [PATCH 0050/2295] updated json_to_yaml.py --- json_to_yaml.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/json_to_yaml.py b/json_to_yaml.py index 078b2fbac..231b6ae1c 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -25,6 +25,8 @@ Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from standard input. +Written to convert old AWS CloudFormation json templates to yaml + See also: https://www.commandlinefu.com/commands/view/12221/convert-json-to-yaml From 375ca823d386750ce6a74e536853fac666c5c27b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:41:32 +0000 Subject: [PATCH 0051/2295] fixed printing from stdin --- json_to_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json_to_yaml.py b/json_to_yaml.py index 231b6ae1c..b3c2b4501 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -56,7 +56,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class JsonToYaml(CLI): @@ -114,7 +114,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.json_to_yaml(sys.stdin.read()) + print(self.json_to_yaml(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From c721b83b7b45431d5f142988861bc8c11d37ba9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:44:24 +0000 Subject: [PATCH 0052/2295] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 095ec85f1..956ed23ee 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -24,15 +24,15 @@ cd "$srcdir"; section "JSON => XML" -tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" +#tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" #echo "tmpfile is $tmpfile" -trap "rm -f $tmpfile" $TRAP_SIGNALS +#trap "rm -f $tmpfile" $TRAP_SIGNALS echo "running json_to_xml.py:" -../json_to_xml.py data/test.json | tee "$tmpfile" +../json_to_xml.py data/test.json | tee /dev/stderr | validate_xml.py echo -echo "now validating generated xml:" -../validate_xml.py "$tmpfile" +echo "running json_to_xml.py from stdin:" +../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py echo From faa993267ac7d159888a8e6d00a46bfa19493c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:47:14 +0000 Subject: [PATCH 0053/2295] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 17d15b2c7..3728aa487 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -24,15 +24,14 @@ cd "$srcdir"; section "XML => JSON" +cd .. + for x in simple.xml plant_catalog.xml; do - tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" - trap "rm -f $tmpfile" $TRAP_SIGNALS + x="tests/data/$x" + #tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" + #trap "rm -f $tmpfile" $TRAP_SIGNALS echo "running xml_to_json.py on $x": - ../xml_to_json.py "data/$x" > "$tmpfile" - echo - echo "now validating generated json": - ../validate_json.py "$tmpfile" + ./xml_to_json.py "$x" | tee /dev/stderr | ./validate_json.py echo - rm -f "$tmpfile" echo "=========" done From df50404af140b8dfbfc144fd8364ef754f29d90f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:55:21 +0000 Subject: [PATCH 0054/2295] removed duplicate STDIN print --- validate_json.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/validate_json.py b/validate_json.py index aab399e41..03cc0aa7a 100755 --- a/validate_json.py +++ b/validate_json.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.11.0' +__version__ = '0.11.1' class JsonValidatorTool(CLI): @@ -297,8 +297,6 @@ def check(self, filename): if filename == '-': filename = '' self.filename = filename - self.valid_json_msg = '{0} => JSON OK'.format(filename) - self.invalid_json_msg = '{0} => JSON INVALID'.format(filename) single_quotes = '(found single quotes not double quotes)' self.valid_json_msg_single_quotes = '{0} {1}'.format(self.valid_json_msg, single_quotes) self.invalid_json_msg_single_quotes = '{0} {1}'.format(self.invalid_json_msg, single_quotes) From a541e6523ed9936673b24d8f9b450eafc2718cbf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:57:58 +0000 Subject: [PATCH 0055/2295] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 3728aa487..5a1b6fffc 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -26,12 +26,18 @@ section "XML => JSON" cd .. -for x in simple.xml plant_catalog.xml; do - x="tests/data/$x" - #tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" - #trap "rm -f $tmpfile" $TRAP_SIGNALS - echo "running xml_to_json.py on $x": - ./xml_to_json.py "$x" | tee /dev/stderr | ./validate_json.py - echo - echo "=========" -done +testdata="tests/data/simple.xml" + +echo "running xml_to_json.py on $testdata": +./xml_to_json.py "$testdata" | tee /dev/stderr | ./validate_json.py +echo + +echo "running xml_to_json.py on stdin < $testdata": +./xml_to_json.py < "$testdata" | tee /dev/stderr | ./validate_json.py +echo + +echo "running xml_to_json.py on tests/data/plant_catalog.xml": +./xml_to_json.py "tests/data/plant_catalog.xml" | ./validate_json.py +echo +echo "XML to JSON tests succeeded!" +echo From 9b7ebae047aa3482616c52a44c27109ead869f7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:24 +0000 Subject: [PATCH 0056/2295] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 956ed23ee..320cb86e8 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -36,3 +36,6 @@ echo echo "running json_to_xml.py from stdin:" ../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py echo + +echo "JSON to XML tests succeeded!" +echo From f49bc2f9332b3cb9349a012d14d1203aeebe444a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:43 +0000 Subject: [PATCH 0057/2295] fixed printing from stdin input --- json_to_xml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json_to_xml.py b/json_to_xml.py index ca2afcafb..03f5b27fa 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -52,7 +52,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class JsonToXml(CLI): @@ -110,7 +110,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.json_to_xml(sys.stdin.read()) + print(self.json_to_xml(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From a0ef0b9b60d992755ebf824e57df8b477c84ee99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:52 +0000 Subject: [PATCH 0058/2295] fixed printing from stdin input --- xml_to_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xml_to_json.py b/xml_to_json.py index 10484bfce..a82ea8adf 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -51,7 +51,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class XmlToJson(CLI): @@ -116,7 +116,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.xml_to_json(sys.stdin.read()) + print(self.xml_to_json(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From 25756d0de5ecc23b7a23d577f89434469900c448 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:15:24 +0000 Subject: [PATCH 0059/2295] added tests/test_xml_to_yaml.sh --- tests/test_xml_to_yaml.sh | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 tests/test_xml_to_yaml.sh diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh new file mode 100755 index 000000000..6f93a6ddf --- /dev/null +++ b/tests/test_xml_to_yaml.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "$0")" && pwd)" + +cd "$srcdir"; + +# shellcheck disable=SC1091 +. utils.sh + +# shellcheck disable=SC1091 +. ../bash-tools/lib/utils.sh + +section "XML => YAML" + +cd .. + +testdata="tests/data/simple.xml" + +echo "running xml_to_yaml.py on $testdata": +./xml_to_yaml.py "$testdata" | tee /dev/stderr | ./validate_yaml.py +echo + +echo "running xml_to_yaml.py on stdin < $testdata": +./xml_to_yaml.py < "$testdata" | tee /dev/stderr | ./validate_yaml.py +echo + +echo "running xml_to_yaml.py on tests/data/plant_catalog.xml": +./xml_to_yaml.py "tests/data/plant_catalog.xml" | ./validate_yaml.py +echo +echo "XML to yaml tests succeeded!" +echo From 49afa470df18b72502864bc0fd59aab7df74fdbb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:15:35 +0000 Subject: [PATCH 0060/2295] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 5a1b6fffc..8d01e3750 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "XML => JSON" From f4b327709c1efdd570a89beb1c10789d23f388e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:16:52 +0000 Subject: [PATCH 0061/2295] updated test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh index 47c39af6e..fc2b58868 100755 --- a/tests/test_json_to_yaml.sh +++ b/tests/test_json_to_yaml.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "JSON => YAML" @@ -29,7 +32,9 @@ cd .. tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" #echo "tmpfile is $tmpfile" -trap "rm -f $tmpfile" $TRAP_SIGNALS +# want var splitting +# shellcheck disable=SC2086 +trap 'rm -f "$tmpfile"' $TRAP_SIGNALS for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do echo "running json_to_yaml.py $x" From 94e0e8848ca944b8103ae6c618d220b1dfeb4f3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:17:11 +0000 Subject: [PATCH 0062/2295] added xml_to_yaml.py --- xml_to_yaml.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 xml_to_yaml.py diff --git a/xml_to_yaml.py b/xml_to_yaml.py new file mode 100755 index 000000000..6643a970a --- /dev/null +++ b/xml_to_yaml.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 18:19:34 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to convert XML to YAML + +Reads any given files as XML and prints the equivalent YAML to stdout for piping or redirecting to a file. + +Directories if given are detected and recursed, processing all files in the directory tree ending in a .xml suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import json +import os +import re +import sys +import xml +import xmltodict +import yaml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, ERRORS, log, log_option + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class XmlToYaml(CLI): + + def __init__(self): + # Python 2.x + super(XmlToYaml, self).__init__() + # Python 3.x + # super().__init__() + self.indent = None + self.re_xml_suffix = re.compile(r'.*\.xml$', re.I) + + def add_options(self): + self.add_opt('-p', '--pretty', action='store_true', help='Pretty Print the resulting YAML') + + @staticmethod + def xml_to_yaml(content, filepath=None): + try: + _ = xmltodict.parse(content) + except xml.parsers.expat.ExpatError as _: + file_detail = '' + if filepath is not None: + file_detail = ' in file \'{0}\''.format(filepath) + die("Failed to parse XML{0}: {1}".format(file_detail, _)) + # xmltodict returns a unicode OrderedDict so need to make it a plain dict to come out properly not like: + # !!python/object/apply:collections.OrderedDict + yaml_string = yaml.safe_dump(json.loads(json.dumps(_)), encoding='utf-8', sort_keys=True) + return yaml_string + + def run(self): + if self.get_opt('pretty'): + log_option('pretty', True) + self.indent = 4 + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'{}' not found".format(arg)) + sys.exit(ERRORS['WARNING']) + if os.path.isfile(arg): + log_option('file', arg) + elif os.path.isdir(arg): + log_option('directory', arg) + else: + die("path '{}' could not be determined as either a file or directory".format(arg)) + for arg in self.args: + self.process_path(arg) + + def process_path(self, path): + if path == '-' or os.path.isfile(path): + self.process_file(path) + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for filename in files: + filepath = os.path.join(root, filename) + if self.re_xml_suffix.match(filepath): + self.process_file(filepath) + else: + die("failed to determine if path '{}' is a file or directory".format(path)) + + def process_file(self, filepath): + log.debug("processing filepath '%s'", filepath) + if filepath == '-': + filepath = '' + if filepath == '': + print(self.xml_to_yaml(sys.stdin.read())) + else: + with open(filepath) as _: + content = _.read() + print(self.xml_to_yaml(content, filepath=filepath)) + + +if __name__ == '__main__': + XmlToYaml().main() From d6c6972ee676e6af0976913f92dc2c65a8e65961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 15:45:13 +0000 Subject: [PATCH 0063/2295] added days field to output --- aws_users_access_key_age.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 89dbbfa02..5c62e2333 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -50,6 +50,7 @@ import datetime import os import sys +from math import ceil import boto3 libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) @@ -64,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' class AWSUsersAccessKeysAge(CLI): @@ -85,7 +86,6 @@ def process_args(self): if self.age: validate_float(self.age, 'age') self.age = float(self.age) - self.age = self.age * 86400 def run(self): iam = boto3.client('iam') @@ -107,18 +107,21 @@ def process_key(self, keys_response, username): if self.only_active_keys and status != 'Active': continue create_date = access_key_item['CreateDate'] + # already cast to datetime.datetime with tzinfo + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + # removing tzinfo for comparison to avoid below error + # - both areOA UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + age_timedelta = self.now - create_date.replace(tzinfo=None) + age_days = int(ceil(age_timedelta.total_seconds() / 86400.0)) if self.age: - # already cast to datetime.datetime with tzinfo - #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') - # removing tzinfo for comparison to avoid below error - # - both are UTC and this doesn't make much difference anyway - # TypeError: can't subtract offset-naive and offset-aware datetimes - if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: + if age_days < self.age: continue - print('{user:20}\t{status:8}\t{date}'.format( + print('{user:20}\t{status:8}\t{date}\t({days:>3} days)'.format( user=username, status=status, - date=create_date)) + date=create_date, + days=age_days)) if __name__ == '__main__': From b7de1020d80ea318ab93affbbdf6b8c6824f52a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 15:53:17 +0000 Subject: [PATCH 0064/2295] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index ae3557778..b2429b8d6 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -86,10 +86,10 @@ def run(self): for users_response in user_paginator.paginate(): for user_item in users_response['Users']: log.debug(user_item) - self.process_password_age(user_item) + self.process_password_last_used(user_item) log.info('Completed') - def process_password_age(self, user_item): + def process_password_last_used(self, user_item): # already cast to datetime.datetime with tzinfo user = user_item['UserName'] if 'PasswordLastUsed' in user_item: From 8f4944a768c28fe7b61299e7c2ed0f02e342e1cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 18:09:37 +0000 Subject: [PATCH 0065/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 5c62e2333..9d39caf33 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -108,7 +108,7 @@ def process_key(self, keys_response, username): continue create_date = access_key_item['CreateDate'] # already cast to datetime.datetime with tzinfo - #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%dT%H:%M:%S%z') # removing tzinfo for comparison to avoid below error # - both areOA UTC and this doesn't make much difference anyway # TypeError: can't subtract offset-naive and offset-aware datetimes From 05f0e995e64d2e8afd31fcc0f298736e6aa86d00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 18:15:41 +0000 Subject: [PATCH 0066/2295] added aws_users_last_used.py --- aws_users_last_used.py | 151 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 aws_users_last_used.py diff --git a/aws_users_last_used.py b/aws_users_last_used.py new file mode 100755 index 000000000..9e7d47e83 --- /dev/null +++ b/aws_users_last_used.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) +# +# https://github.com/harisekhon/nagios-plugins +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Find AWS IAM user accounts last used age in days using the most recently used among their timestamp and access keys + +Optionally filters to only users > N days old to find old user accounts + +Generates an IAM credential report, then parses it to determine the time since each user's password +and access keys were last used + +Output: + + + +Uses the Boto python library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools + +- https://github.com/harisekhon/devops-bash-tools +- https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import os +import sys +import time +import traceback +from datetime import datetime +from io import StringIO +from math import floor +import boto3 +from botocore.exceptions import ClientError +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class AWSUsersLastUsed(CLI): + + def __init__(self): + # Python 2.x + super(AWSUsersLastUsed, self).__init__() + # Python 3.x + # super().__init__() + self.age = None + self.now = None + self.msg = 'AWSUsersLastUsed msg not defined' + + def add_options(self): + self.add_opt('-a', '--age', type=float, + help='Filters to show only accounts last used more than N days ago') + + def process_args(self): + self.no_args() + self.age = self.get_opt('age') + if self.age is not None: + validate_int(self.age, 'age') + + def run(self): + iam = boto3.client('iam') + log.info('generating credentials report') + while True: + result = iam.generate_credential_report() + log.debug('%s', result) + if result['State'] == 'COMPLETE': + log.info('credentials report generated') + break + log.info('waiting for credentials report') + time.sleep(1) + try: + result = iam.get_credential_report() + except ClientError as _: + raise + csv_content = result['Content'] + log.debug('%s', csv_content) + filehandle = StringIO(unicode(csv_content)) + filehandle.seek(0) + csvreader = csv.reader(filehandle) + headers = csvreader.next() + assert headers[0] == 'user' + assert headers[4] == 'password_last_used' + assert headers[10] == 'access_key_1_last_used_date' + assert headers[15] == 'access_key_2_last_used_date' + self.now = datetime.utcnow() + for row in csvreader: + self.process_user(row) + + def process_user(self, row): + log.debug('processing user: %s', row) + user = row[0] + password_last_used = row[4] + access_key_1_last_used_date = row[10] + access_key_2_last_used_date = row[15] + log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', + user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) + min_age = None + for _ in [password_last_used, access_key_1_last_used_date, access_key_2_last_used_date]: + if _ == 'N/A': + continue + # %z not working in Python 2.7 but we already know it's +00:00 + _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + age_timedelta = self.now - _datetime.replace(tzinfo=None) + age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) + if min_age is None or age_days < min_age: + min_age = age_days + if self.age and min_age <= self.age: + return + print('{user:20}\t{days:>3}\t{password_last_used:25}\t'\ + .format(user=user, + days=min_age, + password_last_used=password_last_used) + + '{access_key_1_last_used_date:25}\t{access_key_2_last_used_date:25}'\ + .format(access_key_1_last_used_date=access_key_1_last_used_date, + access_key_2_last_used_date=access_key_2_last_used_date)) + + +if __name__ == '__main__': + AWSUsersLastUsed().main() From 81edcbdca1271cfb8b153f2284de551003b36470 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 15:30:10 +0000 Subject: [PATCH 0067/2295] imported hive_tables_row_counts.py from a while ago --- hive_tables_row_counts.py | 201 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100755 hive_tables_row_counts.py diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py new file mode 100755 index 000000000..2f834a401 --- /dev/null +++ b/hive_tables_row_counts.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a HiveServer2 and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Gets row counts for all Hive tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + get_row_counts(conn, args, database, table, partition_regex) + except impala.error.OperationalError as _: + log.error(_) + +def get_row_counts(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) + except impala.error.OperationalError as _: + if 'is not a partitioned table' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + + +if __name__ == '__main__': + main() From 6657ca03d48caa88f36fdefcc015f7bb7b1d0386 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 15:42:31 +0000 Subject: [PATCH 0068/2295] quick fork of hive_tables_row_counts.py --- impala_tables_row_counts.py | 199 ++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 100755 index 000000000..de4545a41 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala node and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.impalaserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Gets row counts for all impala tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impala node ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='impala', + help='Service principal (default: \'impala\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + get_row_counts(conn, args, database, table, partition_regex) + except impala.error.OperationalError as _: + log.error(_) + +def get_row_counts(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + if 'Table is not partitioned' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + + +if __name__ == '__main__': + main() From ed75ff7c9b989c0fddb81c45d979d1bd7ec7df8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:02:41 +0000 Subject: [PATCH 0069/2295] added hive_foreach_table.py --- hive_foreach_table.py | 183 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 hive_foreach_table.py diff --git a/hive_foreach_table.py b/hive_foreach_table.py new file mode 100755 index 000000000..8fcb5b429 --- /dev/null +++ b/hive_foreach_table.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to connect to a HiveServer2 and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' + +or just for today's partition: + +hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d')) COMPUTE STATS" + + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Hive table") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + query = args.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = args.query.format(table=table) + try: + log.info("running %s", query) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute(query) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + log.error(_) + + +if __name__ == '__main__': + main() From 2573d1b1bb1958be507fd82c9d6060ed709c5ff7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:04:02 +0000 Subject: [PATCH 0070/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index de4545a41..616e00e3f 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -27,7 +27,7 @@ If you get an error like this: -ERROR:impala.impalaserver2:Failed to open transport (tries_left=1) +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) ... TTransportException: TSocket read 0 bytes From de61d68af99584c5fabe386a54e1c6bc31acdf90 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:08:34 +0000 Subject: [PATCH 0071/2295] added impala_foreach_table.py --- impala_foreach_table.py | 181 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 100755 index 000000000..e0bb42856 --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to connect to a Impala and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + +or just for today's partition: + +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d'))" + + +Tested on CDH 5.10, Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Impala table") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impala host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='impala', + help='Service principal (default: \'impala\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + query = args.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = args.query.format(table=table) + try: + log.info("running %s", query) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute(query) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + log.error(_) + + +if __name__ == '__main__': + main() From fb59b03655126a176c369f0696b4806f0b0d577e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:10:52 +0000 Subject: [PATCH 0072/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 760534d27..73aae4c85 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query of statement for each matching table + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From dcaee727df1846f539631a5fac96f560150457e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:12:00 +0000 Subject: [PATCH 0073/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73aae4c85..105e8ae0a 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query of statement for each matching table + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. From 4182da460f1bd41d4e53f22cad1e96feb2cf6ea5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:23:53 +0000 Subject: [PATCH 0074/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index e0bb42856..7fcdc1814 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -171,10 +171,14 @@ def main(): # doesn't support parameterized query quoting from dbapi spec table_cursor.execute(query) for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) + #except impala.error.ProgrammingError as _: + # # COMPUTE STATS returns no results + # if not 'Trying to fetch results on an operation with no results' in _: + # raise if __name__ == '__main__': From 12c059751ca251c9c9a00ea854da287a9b91d542 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:24:33 +0000 Subject: [PATCH 0075/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 8fcb5b429..c05225619 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -173,10 +173,14 @@ def main(): # doesn't support parameterized query quoting from dbapi spec table_cursor.execute(query) for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) + #except impala.error.ProgrammingError as _: + # # COMPUTE STATS returns no results + # if not 'Trying to fetch results on an operation with no results' in _: + # raise if __name__ == '__main__': From 5bed4a290cafa2e263d979519ee3240afdf022ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:06 +0000 Subject: [PATCH 0076/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 7fcdc1814..e966475c2 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a Impala and execute a query for all tables in all databases, +Tool to connect to an Impala node and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From 39bf4033f8841372621c1260e970021cf012022d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:50 +0000 Subject: [PATCH 0077/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index c05225619..7255c5097 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -28,7 +28,7 @@ or just for today's partition: -hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d')) COMPUTE STATS" +hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" Tested on CDH 5.10, Hive 1.1.0 with Kerberos From 4c4f98c68578bb343e7d44b979852f575b2f811e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:55 +0000 Subject: [PATCH 0078/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index e966475c2..b95b5dbbe 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -28,7 +28,7 @@ or just for today's partition: -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d'))" +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" Tested on CDH 5.10, Impala 2.7.0 with Kerberos From 6793ccb767b0944d0d538be60d68006932214ff2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:59:18 +0000 Subject: [PATCH 0079/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index b95b5dbbe..506bbcefb 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -153,8 +153,14 @@ def main(): with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): @@ -168,17 +174,19 @@ def main(): query = args.query.format(table=table) try: log.info("running %s", query) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute(query) - for result in table_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) - #except impala.error.ProgrammingError as _: - # # COMPUTE STATS returns no results - # if not 'Trying to fetch results on an operation with no results' in _: - # raise + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': From 5fa52c0313ca40ee8eb2a0a2c83a13a97d034d82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:03:04 +0000 Subject: [PATCH 0080/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 7255c5097..b15bb3bff 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -63,7 +63,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -153,10 +153,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): @@ -170,18 +176,23 @@ def main(): query = args.query.format(table=table) try: log.info("running %s", query) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute(query) - for result in table_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) - #except impala.error.ProgrammingError as _: - # # COMPUTE STATS returns no results - # if not 'Trying to fetch results on an operation with no results' in _: - # raise + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From b64496eb1d1e5b7de2abf54e6f8324b55d44b130 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:03:08 +0000 Subject: [PATCH 0081/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 506bbcefb..c868ba808 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -63,7 +63,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -151,9 +151,9 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) table_cursor.execute('use {}'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: From 18b8c75f73372262c13721e38c5e24649d872c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:04:58 +0000 Subject: [PATCH 0082/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index c868ba808..bb730f8aa 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -190,4 +190,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From 3ba56bcf1e30529446ff50ab8c3165c4b75db036 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:17 +0000 Subject: [PATCH 0083/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 3058e28dc..ed9d58973 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -199,4 +199,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From a4d996e1d89e74f353243cf64dd1218162d0a24a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:28 +0000 Subject: [PATCH 0084/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 2f834a401..f474e6b93 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -198,4 +198,7 @@ def get_row_counts(conn, args, database, table, partition_regex): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From e66e7d485d38fea3972722ee86d5a2c99e070d40 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:49 +0000 Subject: [PATCH 0085/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 616e00e3f..6dcc0a259 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -196,4 +196,7 @@ def get_row_counts(conn, args, database, table, partition_regex): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From 6f963803faedd4a085b6237ff92f7d7ef7dcf613 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:20:19 +0000 Subject: [PATCH 0086/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index f474e6b93..4093fd8bd 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -142,10 +142,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): From dce68d1406831b763437168fdff44cfa29662597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:20:28 +0000 Subject: [PATCH 0087/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 6dcc0a259..c9bb01d5f 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -140,10 +140,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): From c2de8223bacbe95e5d15348bcd64cfd6787d51a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:30:55 +0000 Subject: [PATCH 0088/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index ed9d58973..a0ff49f89 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -67,25 +67,11 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.1' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'IMPALA_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -94,17 +80,40 @@ def getenvs(keys, default=None): return default def parse_args(): + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] parser = argparse.ArgumentParser( description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ help='HiveServer2 / Impala host ' + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 / Impala port (default: 10000 if called as hive, ' + \ - '21050 if called as impala, $' + \ + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='HiveServer2 / Impala port (default: {}, '.format(default_port) + \ ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: @@ -121,13 +130,6 @@ def parse_args(): if args.verbose > 1 or os.getenv('DEBUG'): log.setLevel(logging.DEBUG) - if 'impala' in sys.argv[0]: - if args.krb5_service_name == 'hive': - log.info('called as impala, setting service principal to impala') - args.krb5_service_name = 'impala' - if args.port == 10000: - log.info('called as impala, setting port to 21050') - args.port = 21050 return args def connect_db(args, database): From 9034a083146cb7d99eab558d2cea47b41e6374b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:38:49 +0000 Subject: [PATCH 0089/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index a0ff49f89..e780c5e69 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -114,7 +114,7 @@ def parse_args(): ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: # TypeError: "delimiter" must be string, not unicode From 969419bb81d169cce11136ff24583a6a8223903f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:41:27 +0000 Subject: [PATCH 0090/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index e780c5e69..f93bb9137 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -80,6 +80,7 @@ def getenvs(keys, default=None): return default def parse_args(): + name = 'HiveServer2' default_port = 10000 default_service_name = 'hive' host_envs = [ @@ -94,6 +95,7 @@ def parse_args(): ] if 'impala' in sys.argv[0]: + name = 'Impala' default_port = 21050 default_service_name = 'impala' host_envs = [ @@ -105,12 +107,12 @@ def parse_args(): 'PORT' ] parser = argparse.ArgumentParser( - description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") + description="Dumps all {} schemas, tables, columns and types to CSV format on stdout".format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 / Impala host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='HiveServer2 / Impala port (default: {}, '.format(default_port) + \ + help='{} port (default: {}, '.format(name, default_port) + \ ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default=default_service_name, From de6b88190d37839f43a85d3e57c2fba32ca74c41 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:43:08 +0000 Subject: [PATCH 0091/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 61 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 4093fd8bd..fef2194e2 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -16,10 +16,10 @@ """ -Connect to a HiveServer2 and get rows counts for all tables in all databases, +Connect to a HiveServer2 / Impala node and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes -Tested on CDH 5.10, Hive 1.1.0 with Kerberos +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -56,18 +56,6 @@ logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -76,19 +64,47 @@ def getenvs(keys, default=None): return default def parse_args(): + name = 'HiveServer2' + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + name = 'Impala' + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] parser = argparse.ArgumentParser( - description="Gets row counts for all Hive tables/partitions matching database / table / partition regexes") + description="Gets row counts for all {} tables / partitions matching database / table / partition regexes"\ + .format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='{} port (default: {}, '.format(name, default_port) + \ + ', $'.join(port_envs) + ')') parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\')') + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -190,7 +206,10 @@ def get_row_counts(conn, args, database, table, partition_regex): print('{db}.{table}.{key}={value}\t{row_count}'.format(\ db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) except impala.error.OperationalError as _: - if 'is not a partitioned table' not in str(_): + # Hive exception msg: is not a partitioned table + # Impala exception msg: Table is not partitioned + if 'is not a partitioned table' not in str(_) and \ + 'Table is not partitioned' not in str(_): raise log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", database, table) From 07549144f1e9b95e8dc2ac357e270cf18d65a56a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:43:16 +0000 Subject: [PATCH 0092/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 209 +----------------------------------- 1 file changed, 1 insertion(+), 208 deletions(-) mode change 100755 => 120000 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py deleted file mode 100755 index c9bb01d5f..000000000 --- a/impala_tables_row_counts.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:et -# -# Author: Hari Sekhon -# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) -# -# https://github.com/harisekhon/devops-python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn -# and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/harisekhon -# - -""" - -Connect to an Impala node and get rows counts for all tables in all databases, -or only those matching given db / table / partition value regexes - -Tested on CDH 5.10, Impala 2.7.0 with Kerberos - -Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see - -https://github.com/cloudera/impyla/issues/286 - -If you get an error like this: - -ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) -... -TTransportException: TSocket read 0 bytes - -then check your --kerberos and --ssl settings match the cluster's settings -(Thrift and Kerberos have the worst error messages ever) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import logging -import os -import re -import socket -import sys -import impala -from impala.dbapi import connect - -__author__ = 'Hari Sekhon' -__version__ = '0.2.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -host_envs = [ - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'IMPALA_PORT', - 'PORT' -] - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - parser = argparse.ArgumentParser( - description="Gets row counts for all impala tables/partitions matching database / table / partition regexes") - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='Impala node ' + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), - help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='impala', - help='Service principal (default: \'impala\')') - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - get_row_counts(conn, args, database, table, partition_regex) - except impala.error.OperationalError as _: - log.error(_) - -def get_row_counts(conn, args, database, table, partition_regex): - log.info("getting partitions for database '%s' table '%s'", database, table) - with conn.cursor() as partition_cursor: - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) - try: - partition_cursor.execute('show partitions {table}'.format(table=table)) - for partitions_row in partition_cursor: - partition_key = partitions_row[0] - partition_value = partitions_row[1] - if not partition_regex.match(partition_value): - log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + - "value does not match regex '%s'", - database, - table, - partition_key, - partition_value, - args.partition) - continue - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ - .format(db=database, table=table, key=partition_key, value=partition_value)) - for result in partition_cursor: - row_count = result[0] - print('{db}.{table}.{key}={value}\t{row_count}'.format(\ - db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - if 'Table is not partitioned' not in str(_): - raise - log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", - database, table) - with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) - for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 120000 index 000000000..65e53c4f2 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1 @@ +hive_tables_row_counts.py \ No newline at end of file From ee04d170bf2adc3f7f9261e6ecfaa8da755e6a9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:46:12 +0000 Subject: [PATCH 0093/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index fef2194e2..80172c21a 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -205,9 +205,9 @@ def get_row_counts(conn, args, database, table, partition_regex): row_count = result[0] print('{db}.{table}.{key}={value}\t{row_count}'.format(\ db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except impala.error.OperationalError as _: - # Hive exception msg: is not a partitioned table - # Impala exception msg: Table is not partitioned + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # Hive impala.error.HiveServer2Error: is not a partitioned table + # Impala impala.error.HiveServer2Error: Table is not partitioned if 'is not a partitioned table' not in str(_) and \ 'Table is not partitioned' not in str(_): raise From 27fae644a72b331dfa6d5950c9b618f6a1f518d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:50:16 +0000 Subject: [PATCH 0094/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 60 ++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index b15bb3bff..e52eca238 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a HiveServer2 and execute a query for all tables in all databases, +Tool to connect to a HiveServer2 / Impala node and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: @@ -26,12 +26,17 @@ hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + or just for today's partition: hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" + -Tested on CDH 5.10, Hive 1.1.0 with Kerberos +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -68,18 +73,6 @@ logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -88,19 +81,46 @@ def getenvs(keys, default=None): return default def parse_args(): - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Hive table") + name = 'HiveServer2' + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + name = 'Impala' + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching {} table".format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='{} port (default: {}, '.format(name, default_port) + \ + ', $'.join(port_envs) + ')') parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ ' (replaces {db} and {table} in the query string with each table and its database)') parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\')') + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From 5fece61317c1c5dfd3486f74b38dce28d41615ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:50:27 +0000 Subject: [PATCH 0095/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 197 +--------------------------------------- 1 file changed, 1 insertion(+), 196 deletions(-) mode change 100755 => 120000 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py deleted file mode 100755 index bb730f8aa..000000000 --- a/impala_foreach_table.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:et -# -# Author: Hari Sekhon -# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) -# -# https://github.com/harisekhon/devops-python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn -# and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/harisekhon -# - -""" - -Tool to connect to an Impala node and execute a query for all tables in all databases, -or only those matching given db / table regexes - -Useful for getting row counts of all tables or analyzing tables: - -eg. - -impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' -impala_foreach_table.py --query 'COMPUTE STATS {table}' - -or just for today's partition: - -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" - - -Tested on CDH 5.10, Impala 2.7.0 with Kerberos - -Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see - -https://github.com/cloudera/impyla/issues/286 - -If you get an error like this: - -ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) -... -TTransportException: TSocket read 0 bytes - -then check your --kerberos and --ssl settings match the cluster's settings -(Thrift and Kerberos have the worst error messages ever) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import logging -import os -import re -import socket -import sys -import impala -from impala.dbapi import connect - -__author__ = 'Hari Sekhon' -__version__ = '0.2.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -host_envs = [ - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'IMPALA_PORT', - 'PORT' -] - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Impala table") - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='Impala host ' + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), - help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') - parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='impala', - help='Service principal (default: \'impala\')') - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - query = args.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = args.query.format(table=table) - try: - log.info("running %s", query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 120000 index 000000000..44716e6dc --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1 @@ +hive_foreach_table.py \ No newline at end of file From 28b4d4c0e46bf2ffc14d66d0c3c8c7f2e884970e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 11:21:01 +0000 Subject: [PATCH 0096/2295] increased timeout --- aws_users_last_used.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 9e7d47e83..eb2a1f963 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -77,6 +77,7 @@ def __init__(self): # super().__init__() self.age = None self.now = None + self.timeout_default = 300 self.msg = 'AWSUsersLastUsed msg not defined' def add_options(self): From 3bb4c74e84e2db5fa189a709f84d836866b15d79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:24:47 +0000 Subject: [PATCH 0097/2295] added AWS Access Key, Secret Key and STS token support, improved ARN matching, moved AWS matches to end as tokens are more generic --- anonymize.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/anonymize.py b/anonymize.py index aae935b51..15e54eb4c 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.7' +__version__ = '0.10.0' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -114,7 +114,6 @@ def __init__(self): self.hash_salt = None # order of iteration of application matters because we must do more specific matches before less specific ones self.anonymizations = OrderedDict([ - ('aws', False), ('ip_prefix', False), ('ip', False), ('subnet_mask', False), @@ -130,11 +129,12 @@ def __init__(self): ('screenos', False), ('junos', False), ('network', False), + ('windows', False), + ('aws', False), # access key, secret key, sts tokens etc are very generic so do them later ('fqdn', False), ('domain', False), ('hostname', False), #('proxy', False), - ('windows', False), ('custom', False), ]) self.exceptions = { @@ -287,10 +287,15 @@ def __init__(self): # arn:partition:service:region:account-id:resource-type/resource-id # arn:partition:service:region:account-id:resource-type:resource-id # eg. arn:aws:iam::123456789012:group/Development/product_1234/* - 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/-]+', + 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/.-]+', # arn:aws:s3:::my_corporate_bucket/Development/* #'aws2': r'\b(arn:aws:s3:::)[^/]+', - 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/-]+', + 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/.-]+', + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html + 'aws3': r'\bAKIA[A-Za-z0-9]{16}\b', # access key + 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key + 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -398,6 +403,10 @@ def __init__(self): # arn:partition:service:region:account-id:resource-type:resource-id 'aws': r'\1\2<\3>', 'aws2': r'\1:', + 'aws3': r'', + 'aws4': r'', + 'aws5': r'', + 'aws6': r'', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -476,7 +485,7 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') - self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations'), + self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ From 05b9b5d7771c4485798c1ee38e998be40b81eb5f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:27:19 +0000 Subject: [PATCH 0098/2295] added aws access key, secret key and sts token tests --- tests/test_anonymize.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index fb0b8036a..53c641477 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -437,17 +437,32 @@ src[114]="arn:aws:iam::123456789012:group/hari" dest[114]="arn:aws:iam:::group/" src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" -dest[115]="arn:aws:iam:::user/*" +dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[116]="arn:aws:iam:::group/*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group/*" +dest[116]="arn:aws:iam:::group//*" src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" dest[117]="arn:aws:s3:::*" +src[118]="AKIAIOSFODNN7EXAMPLE" +dest[118]="" + +src[119]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +dest[119]="" + +src[120]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" +dest[120]="" + +src[121]="ASIAIOSFODNN7EXAMPLE" +dest[121]="" + +src[122]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" +dest[122]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 997494885a75496d9da033fc4613f64cb78c47ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:27:28 +0000 Subject: [PATCH 0099/2295] added .gitallowed --- .gitallowed | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitallowed diff --git a/.gitallowed b/.gitallowed new file mode 100644 index 000000000..bb1dbfb76 --- /dev/null +++ b/.gitallowed @@ -0,0 +1,9 @@ +AKIAIOSFODNN7EXAMPLE + +wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE + +ASIAIOSFODNN7EXAMPLE + +AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== From 00b28e50e5fafae21912f1ea311ad3cc41e6de47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:44:00 +0000 Subject: [PATCH 0100/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 105e8ae0a..64f5ac05e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Environment variables are supported for convenience and also to hide credentials - hostnames / domains / FQDNs - email addresses - IP + MAC addresses - - AWS ARNs + - AWS Access Keys, Secret Keys, ARNs, STS tokens - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings From c0389f8f71f844e4d7b157164b506f4af9464fd5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:49:22 +0000 Subject: [PATCH 0101/2295] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index b2429b8d6..7e00ee4e0 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -16,7 +16,7 @@ """ -Lists all AWS IAM users keys along with the dates and days since their passwords were last used, optionally filtering +Lists all AWS IAM users dates since their passwords were last used, optionally filtering for users whose passwords haven't been used in > N days Output format is: From caf1a045309bcd92626587ebb9d20a8a5abc1bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:49:59 +0000 Subject: [PATCH 0102/2295] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 64f5ac05e..84567c19b 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ Environment variables are supported for convenience and also to hide credentials - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) +- [AWS](https://aws.amazon.com/): + - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days + - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Hadoop](http://hadoop.apache.org/) & NoSQL: - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter From dfe418cab5bc8a6ac9ec679b9370833bbd02ac39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:57:00 +0000 Subject: [PATCH 0103/2295] updated README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84567c19b..f46a7cdfe 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) -### Hadoop, Spark / PySpark, HBase, Pig, Ambari, IPython and Linux Tools ### +### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### -A few of the Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. +A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins. Hari Sekhon -Big Data Contractor, United Kingdom +Cloud Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon From 28aee028a1acd9ad43bfface188365316923e719 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:57:19 +0000 Subject: [PATCH 0104/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f46a7cdfe..99f03ce49 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/pe Hari Sekhon -Cloud Big Data Contractor, United Kingdom +Cloud & Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon From 1ebfa94057253ba63a65d431d96fb3e6a7872610 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 19:00:53 +0000 Subject: [PATCH 0105/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99f03ce49..5e1ae26e2 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc. -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies * [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. From 5abdfebf9784cf7790f2a2f809e06b970f755da4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 19:02:28 +0000 Subject: [PATCH 0106/2295] Update README.md commented out spark elasticsearch repo --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5e1ae26e2..e316eba4e 100644 --- a/README.md +++ b/README.md @@ -348,7 +348,9 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library + You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: From 56a977bae7dbca69764d066f338bf6b5309ce066 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Dec 2019 18:26:54 +0000 Subject: [PATCH 0107/2295] updated aws_users_last_used.py --- aws_users_last_used.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index eb2a1f963..2657f03f7 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -23,6 +23,8 @@ Generates an IAM credential report, then parses it to determine the time since each user's password and access keys were last used +Requires iam:GenerateCredentialReport on resource: * + Output: From c4b18f924182c791483187f97aaf3e5df1e12899 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:18:24 +0000 Subject: [PATCH 0108/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c6a900361..270183668 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c6a9003610ddd2321dc28c2c7d69f98eb43ba7c7 +Subproject commit 270183668565e51f5c31fcf969c6cb57b07e0bfe From 389c4e603421747d2663228a3eed7a641cbbd48e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:18:47 +0000 Subject: [PATCH 0109/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cf0435d86..f3fb9b2a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cf0435d8687fc0951c91ed5cb410a84bce31aca0 +Subproject commit f3fb9b2a9dedb8a4ed7e85183f1739ad2a63fd92 From fcd449247e0ca0957250b11df6e07062aed9ccd8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:20:51 +0000 Subject: [PATCH 0110/2295] switched to more forgiving safe_load_all() --- validate_yaml.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/validate_yaml.py b/validate_yaml.py index 285f75ef8..819943294 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -49,7 +49,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.2' +__version__ = '0.9.3' class YamlValidatorTool(CLI): @@ -86,7 +86,7 @@ def is_excluded(self, path): return False def check_yaml(self, content): - if isYaml(content): + if isYaml(content, safe_load_all=True): if self.get_opt('print'): print(content, end='') else: @@ -99,7 +99,6 @@ def check_yaml(self, content): if not self.get_opt('print'): if self.verbose > 2: try: - # TODO: doesn't validate network-policy.yaml in templates containing multiple yaml docs yaml.safe_load_all(content) except yaml.YAMLError as _: print(_) From 319ab52ca5560fad536f651bacf6406b4cd39a6b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 17:58:24 +0000 Subject: [PATCH 0111/2295] updated help.sh --- tests/help.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/help.sh b/tests/help.sh index 851451152..7b4edd786 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -22,7 +22,8 @@ cd "$srcdir/.."; # shellcheck disable=SC1091 . ./tests/utils.sh -for x in $(echo ./*.py 2>/dev/null); do +# shellcheck disable=SC2068 +for x in ${@:-$(echo ./*.py 2>/dev/null)}; do isExcluded "$x" && continue set +e echo "$x --help" @@ -33,7 +34,14 @@ for x in $(echo ./*.py 2>/dev/null); do if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue + [[ "$x" =~ (hive|impala)_foreach_table.py$ ]] && continue + [[ "$x" =~ (hive|impala)_tables_row_counts.py$ ]] && continue [[ "$x" =~ pythonpath.py$ ]] && continue + elif [ $status = 1 ]; then + if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && + ! python -c 'import krbV'; then # best effort, not available on Mac any more + continue + fi fi [ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; } done From e45d8a1fd2b970855cf306d5218cf5a3d9d2fb61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 18:14:08 +0000 Subject: [PATCH 0112/2295] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index d625bf2db..73b8bf7b5 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -28,7 +28,9 @@ section "Testing DockerHub Show Tags" check './dockerhub_search.py centos' "DockerHub Search for CentOS" check './dockerhub_search.py harisekhon' "DockerHub Search for harisekhon" check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon -n 30" -check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" +# this no longer works, API must have changed +#check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" +check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" From a730b7a3956210946eb6d5a497eed24e6286c08a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Dec 2019 13:28:33 +0000 Subject: [PATCH 0113/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 270183668..384e5f8b3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 270183668565e51f5c31fcf969c6cb57b07e0bfe +Subproject commit 384e5f8b366fdeb6a73363dbf388824f5de5e7bc From a852ffa341d9b8a6b2945ee32f56f83bd48b9085 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 17:34:42 +0000 Subject: [PATCH 0114/2295] added hexanonymize.py --- hexanonymize.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100755 hexanonymize.py diff --git a/hexanonymize.py b/hexanonymize.py new file mode 100755 index 000000000..600083615 --- /dev/null +++ b/hexanonymize.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to anonymize hex input but keeping the structure of the positions of numbers and digits + +Useful to retain the structure of ID formats + +Reads any given files or standard input and replaces each hex character with an incrementing number of letter (a-f) +printing to stdout for piping or redirecting to a file as per unix filter command standards + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class HexAnonymize(CLI): + + def __init__(self): + # Python 2.x + super(HexAnonymize, self).__init__() + # Python 3.x + # super().__init__() + self.preserve_case = False + self.only_hex_alphas = False + + def add_options(self): + super(HexAnonymize, self).add_options() + self.add_opt('-c', '--case', action='store_true', help='Preserve case') + self.add_opt('-o', '--only-hex', action='store_true', + help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety') + + def process_options(self): + super(HexAnonymize, self).process_options() + self.preserve_case = self.get_opt('case') + self.only_hex_alphas = self.get_opt('only_hex') + + def hexanonymize(self, filehandle): + preserve_case = self.preserve_case + only_hex_alphas = self.only_hex_alphas + hex_alphas = ['a', 'b', 'c', 'd', 'e', 'f'] + for line in filehandle: + integer = 1 + letter = 'a' + for char in line: + if char.isdigit(): + char = integer + integer += 1 + if integer > 9: + integer = 0 + elif char.lower() in hex_alphas or (not only_hex_alphas and char.isalpha()): + if preserve_case and char.isupper(): + char = letter.upper() + else: + char = letter + letter = chr(ord(char) + 1) + if letter not in hex_alphas: + letter = 'a' + print(char, end='') + + + def run(self): + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'%s' not found" % arg) + sys.exit(1) + for arg in self.args: + if arg == '-': + self.hexanonymize(sys.stdin) + else: + with open(arg) as filehandle: + self.hexanonymize(filehandle) + + +if __name__ == '__main__': + HexAnonymize().main() From 56ce78a2273edb76dfeaa068fbd2bf6de723fdaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:07:53 +0000 Subject: [PATCH 0115/2295] updated hexanonymize.py --- hexanonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hexanonymize.py b/hexanonymize.py index 600083615..de037485e 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -84,13 +84,13 @@ def hexanonymize(self, filehandle): integer += 1 if integer > 9: integer = 0 - elif char.lower() in hex_alphas or (not only_hex_alphas and char.isalpha()): + elif (not only_hex_alphas and char.isalpha()) or char.lower() in hex_alphas: if preserve_case and char.isupper(): char = letter.upper() else: char = letter letter = chr(ord(char) + 1) - if letter not in hex_alphas: + if letter.lower() not in hex_alphas: letter = 'a' print(char, end='') From 267634f4773ff3191f1fd61e90b8fd7ac6f620cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:08:37 +0000 Subject: [PATCH 0116/2295] added test_hexanonymize.sh --- tests/test_hexanonymize.sh | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 tests/test_hexanonymize.sh diff --git a/tests/test_hexanonymize.sh b/tests/test_hexanonymize.sh new file mode 100755 index 000000000..57da1e297 --- /dev/null +++ b/tests/test_hexanonymize.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-02 17:35:08 +0000 (Thu, 02 Jan 2020) +# +# https://github.com/harisekhon/devop-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$srcdir/.."; + +. ./tests/utils.sh + +section "HexAnonymize" + +start_time=$(date +%s) + +run++ +check_output "abc123456def789012abcd" hexanonymize.py <<< "xyz987654rst654321AKIA" + +run++ +check_output "abc123456def789012ABCD" hexanonymize.py -c <<< "xyz987654rst654321AKIA" + +run++ +check_output "xyz123456rst789012abC" hexanonymize.py -c -o <<< "xyz987654rst654321caD" + +run++ +check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD" + +echo +echo "Total Tests run: $run_count" +time_taken "$start_time" "All version tests for $name completed in" +echo +untrap From c50eebeba4c84f641d4d48c6aa75c01baa876299 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:12:03 +0000 Subject: [PATCH 0117/2295] updated hexanonymize.py --- hexanonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hexanonymize.py b/hexanonymize.py index de037485e..91728e7fc 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -63,13 +63,13 @@ def __init__(self): def add_options(self): super(HexAnonymize, self).add_options() self.add_opt('-c', '--case', action='store_true', help='Preserve case') - self.add_opt('-o', '--only-hex', action='store_true', + self.add_opt('-o', '--hex-only', action='store_true', help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety') def process_options(self): super(HexAnonymize, self).process_options() self.preserve_case = self.get_opt('case') - self.only_hex_alphas = self.get_opt('only_hex') + self.only_hex_alphas = self.get_opt('hex_only') def hexanonymize(self, filehandle): preserve_case = self.preserve_case From 88a6f2b6054294da428b680c288ae7acdf7f49e4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 3 Jan 2020 23:26:35 +0000 Subject: [PATCH 0118/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 53c641477..8c9498024 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -442,26 +442,26 @@ dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[116]="arn:aws:iam:::group/*" -src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group//*" +src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[117]="arn:aws:iam:::group//*" -src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" -dest[117]="arn:aws:s3:::*" +src[118]="arn:aws:s3:::my_corporate_bucket/Development/*" +dest[118]="arn:aws:s3:::*" -src[118]="AKIAIOSFODNN7EXAMPLE" -dest[118]="" +src[119]="AKIAIOSFODNN7EXAMPLE" +dest[119]="" -src[119]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -dest[119]="" +src[120]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +dest[120]="" -src[120]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" -dest[120]="" +src[121]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" +dest[121]="" -src[121]="ASIAIOSFODNN7EXAMPLE" -dest[121]="" +src[122]="ASIAIOSFODNN7EXAMPLE" +dest[122]="" -src[122]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" -dest[122]="" +src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" +dest[123]="" # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 2efd6307651985eda772a23663cd73fd17e78229 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:12 +0000 Subject: [PATCH 0119/2295] renamed hadoop_hdfs_files_native_checksums.jy to hdfs_files_native_checksums.jy --- ...fs_files_native_checksums.jy => hdfs_files_native_checksums.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_files_native_checksums.jy => hdfs_files_native_checksums.jy (100%) diff --git a/hadoop_hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy similarity index 100% rename from hadoop_hdfs_files_native_checksums.jy rename to hdfs_files_native_checksums.jy From 3a00a1e52f75c2948ad0b8cdf2736c7a99062196 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:18 +0000 Subject: [PATCH 0120/2295] renamed hadoop_hdfs_files_stats.jy to hdfs_files_stats.jy --- hadoop_hdfs_files_stats.jy => hdfs_files_stats.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_files_stats.jy => hdfs_files_stats.jy (100%) diff --git a/hadoop_hdfs_files_stats.jy b/hdfs_files_stats.jy similarity index 100% rename from hadoop_hdfs_files_stats.jy rename to hdfs_files_stats.jy From 25725f41def7c2188b2bb0fd3b75aefa10ecb45e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:23 +0000 Subject: [PATCH 0121/2295] renamed hadoop_hdfs_time_block_reads.jy to hdfs_time_block_reads.jy --- hadoop_hdfs_time_block_reads.jy => hdfs_time_block_reads.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_time_block_reads.jy => hdfs_time_block_reads.jy (100%) diff --git a/hadoop_hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy similarity index 100% rename from hadoop_hdfs_time_block_reads.jy rename to hdfs_time_block_reads.jy From 604f0e9c86e37223b484c4e65b400fa21e3ca93e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:30:16 +0000 Subject: [PATCH 0122/2295] updated README.md --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e316eba4e..9d75612da 100644 --- a/README.md +++ b/README.md @@ -105,9 +105,10 @@ Environment variables are supported for convenience and also to hide credentials - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex @@ -257,14 +258,14 @@ make The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured) ``` -hadoop_hdfs_time_block_reads.jy -hadoop_hdfs_files_native_checksums.jy -hadoop_hdfs_files_stats.jy +hdfs_time_block_reads.jy +hdfs_files_native_checksums.jy +hdfs_files_stats.jy ``` Run like so: ``` -jython -J-cp $(hadoop classpath) hadoop_hdfs_time_block_reads.jy --help +jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. @@ -286,7 +287,7 @@ Jython is a simple download and unpack and can be fetched from http://www.jython Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg: ``` -/opt/jython-2.7.0/bin/jython hadoop_hdfs_time_block_reads.jy ... +/opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ... ``` From 1db035361637922ed487f80be6e4313d3eb4253a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:31:20 +0000 Subject: [PATCH 0123/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d75612da..b063a839a 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,7 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The ```-J-cp $(hadoop classpath)``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. See below for procedure to install Jython if you don't already have it. From 1577b0319456095a477676b8124cdc3063a687dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 17:49:07 +0000 Subject: [PATCH 0124/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 8c9498024..c18db1ae6 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -440,7 +440,7 @@ src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group/*" +dest[116]="arn:aws:iam:::group//*" src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[117]="arn:aws:iam:::group//*" @@ -463,6 +463,9 @@ dest[122]="" src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" dest[123]="" +src[124]="sg-5f63c627" +dest[124]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 8573959780d35723f0324f05719a96e9bdabf1fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 17:59:53 +0000 Subject: [PATCH 0125/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c18db1ae6..c1fa8450e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -466,6 +466,9 @@ dest[123]="" src[124]="sg-5f63c627" dest[124]="" +src[125]="s3://myBucket/file.txt" +dest[125]="s3:///file.txt" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 34e192dc078bef5112dc6530764917548fb0746f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 18:00:21 +0000 Subject: [PATCH 0126/2295] added aws security group and bucket anonymization --- anonymize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 15e54eb4c..99e1dd31c 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.0' +__version__ = '0.10.1' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -296,6 +296,8 @@ def __init__(self): 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key + 'aws7': r'\bsg-[a-z0-9]{8}(?!\w)', # security group id + 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -407,6 +409,8 @@ def __init__(self): 'aws4': r'', 'aws5': r'', 'aws6': r'', + 'aws7': r'', + 'aws8': r'\1:///', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From b7867276c5ebe0813e04d4237f91c02e5588adaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 10:27:26 +0000 Subject: [PATCH 0127/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c1fa8450e..c3c0bcd54 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -463,12 +463,26 @@ dest[122]="" src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" dest[123]="" +# security groups src[124]="sg-5f63c627" -dest[124]="" +dest[124]="" src[125]="s3://myBucket/file.txt" dest[125]="s3:///file.txt" +# RDS +src[126]="--db-name myDB" +dest[126]="--db-name " + +src[127]="--db-instance-identifier myDBinstance" +dest[127]="--db-instance-identifier " + +src[128]="--master-user-password blah" +dest[128]="--master-user-password " + +src[129]="--master-username first.last" +dest[129]="--master-username " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From d74d2a0c9bb7b3e5be7c94b64454dee6b31a44eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 10:28:19 +0000 Subject: [PATCH 0128/2295] added --database, change aws security group and cisco password, improved -password matches --- anonymize.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/anonymize.py b/anonymize.py index 99e1dd31c..5c28bf27f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.1' +__version__ = '0.10.2' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -118,6 +118,7 @@ def __init__(self): ('ip', False), ('subnet_mask', False), ('mac', False), + ('db', False), ('kerberos', False), ('email', False), ('password', False), @@ -296,8 +297,10 @@ def __init__(self): 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key - 'aws7': r'\bsg-[a-z0-9]{8}(?!\w)', # security group id + 'aws7': r'\bsg-[A-Za-z0-9]{8}(?).*?$', 'cisco3': r'\ssecret\s.*?$', 'cisco4': r'\smd5\s+.*?$', 'cisco5': r'\scommunity\s+.*$', @@ -409,8 +412,10 @@ def __init__(self): 'aws4': r'', 'aws5': r'', 'aws6': r'', - 'aws7': r'', + 'aws7': r'', 'aws8': r'\1:///', + 'db': r'\1', + 'db2': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -489,7 +494,10 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') - self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations') + self.add_opt('-w', '--aws', action='store_true', + help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)') + self.add_opt('-b', '--db', '--database', action='store_true', + help='Apply database anonymizations (db name, instance name)') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ @@ -600,7 +608,10 @@ def process_options(self): for _ in self.anonymizations: if _ in ('subnet_mask', 'mac', 'group'): continue - self.anonymizations[_] = self.get_opt(_) + elif _ == 'database': + self.anonymizations['db'] = True + else: + self.anonymizations[_] = self.get_opt(_) log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_])) self._process_options_host() self._process_options_network() From 5f3458919739c6eb85412f260fe49664273deea8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:28:21 +0000 Subject: [PATCH 0129/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c3c0bcd54..1de505f57 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -483,6 +483,9 @@ dest[128]="--master-user-password " src[129]="--master-username first.last" dest[129]="--master-username " +src[130]="--schema-name mySchema" +dest[130]="--schema-name " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From a8c25d519e600f0738e1a550fa4b4efde39266db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:28:27 +0000 Subject: [PATCH 0130/2295] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index 5c28bf27f..ac639541a 100755 --- a/anonymize.py +++ b/anonymize.py @@ -301,6 +301,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'db': r'(-{1,2}(?:db|database)?-?name' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), 'db2': r'(-{1,2}(?:db|database)-?instance(-?identifier)?' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), + 'db3': r'(-{1,2}schema-?name' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -416,6 +417,7 @@ def __init__(self): 'aws8': r'\1:///', 'db': r'\1', 'db2': r'\1', + 'db3': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From 122fdb64fad622d70c35c5f63ddb50f92201e5b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:30:05 +0000 Subject: [PATCH 0131/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1de505f57..c55d8637c 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -486,6 +486,9 @@ dest[129]="--master-username " src[130]="--schema-name mySchema" dest[130]="--schema-name " +src[131]="=arn:aws:acm:us-east-1:123456:certificate/abc-123" +dest[131]="=arn:aws:acm:us-east-1::certificate/" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 17bb20ef8a82827bb93675bea3dc9cc84f810718 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:00:24 +0000 Subject: [PATCH 0132/2295] added -key match and improved db regex --- anonymize.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index ac639541a..0214065e6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.2' +__version__ = '0.10.3' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -299,9 +299,10 @@ def __init__(self): 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key 'aws7': r'\bsg-[A-Za-z0-9]{8}(?', 'aws7': r'', 'aws8': r'\1:///', + 'aws9': r'\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From a0b35853a6cc6afec2ca1580454c7052f33d3ca3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:00:28 +0000 Subject: [PATCH 0133/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c55d8637c..70d331acd 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -489,6 +489,12 @@ dest[130]="--schema-name " src[131]="=arn:aws:acm:us-east-1:123456:certificate/abc-123" dest[131]="=arn:aws:acm:us-east-1::certificate/" +src[132]="--key-name my-key" +dest[132]="--key-name " + +src[133]="private-key my-key" +dest[133]="private-key " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 26b27dba68b5f4993b3aaaefc70e1a08c87969c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:36:55 +0000 Subject: [PATCH 0134/2295] updated anonymize.py --- anonymize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 0214065e6..c610901c9 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.3' +__version__ = '0.10.4' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -300,9 +300,9 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(? Date: Thu, 9 Jan 2020 15:10:40 +0000 Subject: [PATCH 0135/2295] refined switch prefix variant, added --cluster variants --- anonymize.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index c610901c9..eafca0e97 100755 --- a/anonymize.py +++ b/anonymize.py @@ -283,6 +283,8 @@ def __init__(self): arg_sep = r'[=\s:]+' # openssl uses -passin switch pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' + # allowing --blah- prefix variants + switch_prefix = r'(?', 'aws8': r'\1:///', 'aws9': r'\1', + 'aws10': r'\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From 777f5bdcf383c8281a4df99c99e97a926c2b9814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 15:10:43 +0000 Subject: [PATCH 0136/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 70d331acd..1dce824c7 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -470,12 +470,11 @@ dest[124]="" src[125]="s3://myBucket/file.txt" dest[125]="s3:///file.txt" -# RDS -src[126]="--db-name myDB" -dest[126]="--db-name " +src[126]="aws rds create-db-instance --db-name myDB" +dest[126]="aws rds create-db-instance --db-name " -src[127]="--db-instance-identifier myDBinstance" -dest[127]="--db-instance-identifier " +src[127]="aws rds modify-db-instance --db-instance-identifier myDBinstance" +dest[127]="aws rds modify-db-instance --db-instance-identifier " src[128]="--master-user-password blah" dest[128]="--master-user-password " @@ -492,8 +491,12 @@ dest[131]="=arn:aws:acm:us-east-1::certificate/" src[132]="--key-name my-key" dest[132]="--key-name " -src[133]="private-key my-key" -dest[133]="private-key " +src[133]="-private-key my-key" +dest[133]="-private-key " + +src[134]="aws elasticache create-cache-cluster --cache-cluster-id myCluster" +dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From b6702dd2a2dd0ae77720cbb6fd7aaeabfe2ac144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 18:08:40 +0000 Subject: [PATCH 0137/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1dce824c7..0fe2ee62c 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -497,6 +497,9 @@ dest[133]="-private-key " src[134]="aws elasticache create-cache-cluster --cache-cluster-id myCluster" dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " +src[135]="subnet-abc12345" +dest[135]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 7cfc78cd5658b084d376ddf42143431603efc9ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 18:09:12 +0000 Subject: [PATCH 0138/2295] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index eafca0e97..637c9b280 100755 --- a/anonymize.py +++ b/anonymize.py @@ -303,6 +303,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'aws9': r'({switch_prefix}key(:?-?name)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), 'aws10': r'({switch_prefix}cluster(?:-?id)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), + 'aws11': r'(?/', 'aws9': r'\1', 'aws10': r'\1', + 'aws11': r'', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From 3c89b46e3a41388333122904769b7cd234063c31 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 10:05:50 +0000 Subject: [PATCH 0139/2295] updated anonymize.py --- anonymize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 637c9b280..28a1c2bcd 100755 --- a/anonymize.py +++ b/anonymize.py @@ -303,7 +303,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'aws9': r'({switch_prefix}key(:?-?name)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), 'aws10': r'({switch_prefix}cluster(?:-?id)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), - 'aws11': r'(? Date: Fri, 10 Jan 2020 10:09:12 +0000 Subject: [PATCH 0140/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 0fe2ee62c..878cac882 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -500,6 +500,9 @@ dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " src[135]="subnet-abc12345" dest[135]="" +src[136]="arn:aws:acm:us-east-1:123456:function:myFunction123:7" +dest[136]="arn:aws:acm:us-east-1::function::7" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 4dd0868fc84c9ff485f7f3c9d73474c0abce5ba7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 13:33:06 +0000 Subject: [PATCH 0141/2295] added --generic --- anonymize.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/anonymize.py b/anonymize.py index 28a1c2bcd..f5db7e4b5 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.4' +__version__ = '0.10.5' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -119,6 +119,7 @@ def __init__(self): ('subnet_mask', False), ('mac', False), ('db', False), + ('generic', False), ('kerberos', False), ('email', False), ('password', False), @@ -285,6 +286,7 @@ def __init__(self): pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' # allowing --blah- prefix variants switch_prefix = r'(?', 'aws7': r'', 'aws8': r'\1:///', - 'aws9': r'\1', - 'aws10': r'\1', - 'aws11': r'', + 'aws9': r'', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', + 'generic': r'\1://', + 'generic2': r'\1', + 'generic3': r'\1', + 'generic4': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -508,6 +514,8 @@ def add_options(self): help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)') self.add_opt('-b', '--db', '--database', action='store_true', help='Apply database anonymizations (db name, instance name)') + self.add_opt('-g', '--generic', action='store_true', + help='Apply generic anonymizations (file://, key, cluster name / id etc)') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ From 30245833319b83af58e0798cdba47b9eace1b596 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 13:33:11 +0000 Subject: [PATCH 0142/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 878cac882..1717579b1 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -503,6 +503,9 @@ dest[135]="" src[136]="arn:aws:acm:us-east-1:123456:function:myFunction123:7" dest[136]="arn:aws:acm:us-east-1::function::7" +src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" +dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 32854e2911b8c1becc16d0e2faf2e3c88d77be85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 15:11:42 +0000 Subject: [PATCH 0143/2295] updated anonymize.py --- anonymize.py | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/anonymize.py b/anonymize.py index f5db7e4b5..2f7fb8a21 100755 --- a/anonymize.py +++ b/anonymize.py @@ -120,6 +120,8 @@ def __init__(self): ('mac', False), ('db', False), ('generic', False), + # kerberos must be applied before email + # - if email is applied first, 'user/host@realm' becomes 'user/', exposing user ('kerberos', False), ('email', False), ('password', False), @@ -304,13 +306,30 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(? ' + \ r'http://:\@. Also works with https://') self.add_opt('-K', '--kerberos', action='store_true', - help=r'Kerberos 5 principals in the form @ or /@ ' + \ + help=r'Apply Kerberos anonymizations eg. @, /@ ' + \ '(where must match a valid domain name - otherwise use --custom and populate ' + \ - r'anonymize_custom.conf). These kerberos principals are anonymizebed to ' + \ - '. There is a special exemption for Hadoop Kerberos principals such ' + \ - 'as NN/_HOST@ which preserves the literal \'_HOST\' instance since that\'s ' + \ - 'useful to know for debugging, the principal and realm will still be anonymizebed in ' + \ - 'those cases (if wanting to retain NN/_HOST then use --domain instead of --kerberos). ' + \ - 'This is applied before --email in order to not prevent the email replacement leaving ' + \ - r'this as user/host\@realm to user/, which would have exposed \'user\'' + \ - '. Auto enables --email, --domain and --fqdn') + r'anonymize_custom.conf). Hadoop principals preserve the generic _HOST placeholder eg. ' + \ + '/_HOST@ (if wanting to retain full prefix eg. NN/_HOST then use ' + \ + '--domain instead of --kerberos). --kerberos auto-enables --email, --domain and --fqdn') self.add_opt('-L', '--ldap', action='store_true', help='Apply LDAP anonymization ' + \ '(~100 attribs eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)') @@ -583,7 +597,7 @@ def add_options(self): # 'should probably also apply --ip and --host if using this. Auto enables --http-auth') self.add_opt('-N', '--network', action='store_true', help='Apply all network anonymization, whether Cisco, ScreenOS, JunOS for secrets, auth, ' + \ - 'usernames, passwords, md5s, PSKs, AS, SNMP etc.') + 'usernames, passwords, md5s, PSKs, AS, SNMP community strings etc.') self.add_opt('-c', '--cisco', action='store_true', help='Apply Cisco IOS/IOS-XR/NX-OS configuration format anonymization') self.add_opt('-s', '--screenos', action='store_true', From e5357fd74400c98d4f747c13bbcbe70f72489632 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 15:14:32 +0000 Subject: [PATCH 0144/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b063a839a..17efe0c68 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Environment variables are supported for convenience and also to hide credentials - Linux: - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - - anonymizes: + - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses - IP + MAC addresses From f0bdacc587ff59355f5526c77dbf6652200ffbaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:07:14 +0000 Subject: [PATCH 0145/2295] updated anonymize.py --- anonymize.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 2f7fb8a21..8210af2d6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.5' +__version__ = '0.10.6' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -318,15 +318,19 @@ def __init__(self): id_or_name=id_or_name, switch_prefix=switch_prefix), 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), - 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})[\w-]+'\ + 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})[\w-]+'\ + 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})[\w-]+'\ + 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})\S+'\ + .format(arg_sep=arg_sep, + id_or_name=id_or_name, + switch_prefix=switch_prefix), + 'generic5': r'({switch_prefix}load-?balancer{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), @@ -451,6 +455,7 @@ def __init__(self): 'generic2': r'\1', 'generic3': r'\1', 'generic4': r'\1', + 'generic5': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From 55f6afc7e430bd085fb131ddda87c89e4c250763 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:07:32 +0000 Subject: [PATCH 0146/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1717579b1..b203357f4 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -506,6 +506,9 @@ dest[136]="arn:aws:acm:us-east-1::function::7" src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" +src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' +dest[138]=' aws elb create-load-balancer --load-balancer-name ...' + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 08eca37ea8184589d9416d1002a704489f944d1d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:23:49 +0000 Subject: [PATCH 0147/2295] added aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 aws_users_unused_access_keys.py diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py new file mode 100755 index 000000000..00440214d --- /dev/null +++ b/aws_users_unused_access_keys.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) +# +# https://github.com/harisekhon/nagios-plugins +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Find AWS IAM user access keys that haven't been used in N days or since created + +Generates an IAM credential report, then parses it to determine the time since each user's password +and access keys were last used + +Requires iam:GenerateCredentialReport on resource: * + +Output: + + + + +Uses the Boto python library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools + +- https://github.com/harisekhon/devops-bash-tools +- https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import os +import sys +import time +import traceback +from datetime import datetime +from io import StringIO +from math import floor +import boto3 +from botocore.exceptions import ClientError +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class AWSUnusedAccessKeys(CLI): + + def __init__(self): + # Python 2.x + super(AWSUnusedAccessKeys, self).__init__() + # Python 3.x + # super().__init__() + self.age = None + self.now = None + self.timeout_default = 300 + self.msg = 'AWSUnusedAccessKeys msg not defined' + + def add_options(self): + self.add_opt('-a', '--age', type=float, default=90, + help='Show only access keys not used in the last N days (default: 90)') + + def process_args(self): + self.no_args() + self.age = self.get_opt('age') + if self.age is not None: + validate_int(self.age, 'age') + + def run(self): + iam = boto3.client('iam') + log.info('generating credentials report') + while True: + result = iam.generate_credential_report() + log.debug('%s', result) + if result['State'] == 'COMPLETE': + log.info('credentials report generated') + break + log.info('waiting for credentials report') + time.sleep(1) + try: + result = iam.get_credential_report() + except ClientError as _: + raise + csv_content = result['Content'] + log.debug('%s', csv_content) + filehandle = StringIO(unicode(csv_content)) + filehandle.seek(0) + csvreader = csv.reader(filehandle) + headers = csvreader.next() + log.debug('headers: %s', headers) + assert headers[0] == 'user' + assert headers[4] == 'password_last_used' + assert headers[10] == 'access_key_1_last_used_date' + assert headers[15] == 'access_key_2_last_used_date' + self.now = datetime.utcnow() + for row in csvreader: + self.process_user(row) + + def process_user(self, row): + log.debug('processing user: %s', row) + user = row[0] + password_last_used = row[4] + access_key_1_last_used_date = row[10] + access_key_2_last_used_date = row[15] + log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', + user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) + key = 1 + for _ in [access_key_1_last_used_date, access_key_2_last_used_date]: + if _ == 'N/A': + continue + # %z not working in Python 2.7 but we already know it's +00:00 + _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + age_timedelta = self.now - _datetime.replace(tzinfo=None) + age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) + if age_days > self.age: + log.debug('access key %s last used %s days ago > %s', key, age_days, self.age) + print('{user:20}\t{key}\t{days:>3}\t{access_key_last_used:25}\t'\ + .format(user=user, + key=key, + days=age_days, + access_key_last_used=_)) + key += 1 + + +if __name__ == '__main__': + AWSUnusedAccessKeys().main() From 9b70c9638f194e1ca8c86baca6670b85057a1ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:25:44 +0000 Subject: [PATCH 0148/2295] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 00440214d..b7fcdf9cd 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -114,7 +114,6 @@ def run(self): headers = csvreader.next() log.debug('headers: %s', headers) assert headers[0] == 'user' - assert headers[4] == 'password_last_used' assert headers[10] == 'access_key_1_last_used_date' assert headers[15] == 'access_key_2_last_used_date' self.now = datetime.utcnow() From 223e005b504a4d89698ed09fba5c7b4102d2b5f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 11:14:22 +0000 Subject: [PATCH 0149/2295] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 76 +++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index b7fcdf9cd..9b9083eb9 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -16,7 +16,7 @@ """ -Find AWS IAM user access keys that haven't been used in N days or since created +Find AWS IAM user access keys that haven't been used in N days and keys that have never been used Generates an IAM credential report, then parses it to determine the time since each user's password and access keys were last used @@ -25,8 +25,7 @@ Output: - - + Uses the Boto python library, read here for the list of ways to configure your AWS credentials: @@ -66,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class AWSUnusedAccessKeys(CLI): @@ -78,12 +77,15 @@ def __init__(self): # super().__init__() self.age = None self.now = None + self.only_active = False self.timeout_default = 300 self.msg = 'AWSUnusedAccessKeys msg not defined' def add_options(self): - self.add_opt('-a', '--age', type=float, default=90, - help='Show only access keys not used in the last N days (default: 90)') + self.add_opt('-a', '--age', type=float, default=30, + help='Show only access keys not used in the last N days (default: 30)') + self.add_opt('-o', '--only-active', action='store_true', + help='Only show access keys that are active') def process_args(self): self.no_args() @@ -114,7 +116,11 @@ def run(self): headers = csvreader.next() log.debug('headers: %s', headers) assert headers[0] == 'user' + assert headers[8] == 'access_key_1_active' + assert headers[9] == 'access_key_1_last_rotated' assert headers[10] == 'access_key_1_last_used_date' + assert headers[13] == 'access_key_2_active' + assert headers[14] == 'access_key_2_last_rotated' assert headers[15] == 'access_key_2_last_used_date' self.now = datetime.utcnow() for row in csvreader: @@ -123,27 +129,53 @@ def run(self): def process_user(self, row): log.debug('processing user: %s', row) user = row[0] - password_last_used = row[4] - access_key_1_last_used_date = row[10] - access_key_2_last_used_date = row[15] - log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', - user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) - key = 1 - for _ in [access_key_1_last_used_date, access_key_2_last_used_date]: - if _ == 'N/A': + access_keys = {1:{}, 2:{}} + access_keys[1]['active'] = row[8] + access_keys[1]['last_used_date'] = row[10] + access_keys[1]['last_rotated'] = row[9] + access_keys[2]['active'] = row[13] + access_keys[2]['last_rotated'] = row[14] + access_keys[2]['last_used_date'] = row[15] + for key in [1, 2]: + active = access_keys[key]['active'] + if not isinstance(active, bool): + assert active in ('true', 'false') + active = active.lower() == 'true' + created = access_keys[key]['last_rotated'] + last_used = access_keys[key]['last_used_date'] + log.debug('user: %s, key: %s, active: %s, created: %s, last_used_date: %s, ', + user, + key, + active, + created, + last_used + ) + if not active and self.only_active: + continue + if last_used == 'N/A': + if created == 'N/A': + continue + self.print_key(user, key, active, 'N/A', last_used, created) continue # %z not working in Python 2.7 but we already know it's +00:00 - _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + _datetime = datetime.strptime(last_used.split('+')[0], '%Y-%m-%dT%H:%M:%S') age_timedelta = self.now - _datetime.replace(tzinfo=None) age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) if age_days > self.age: - log.debug('access key %s last used %s days ago > %s', key, age_days, self.age) - print('{user:20}\t{key}\t{days:>3}\t{access_key_last_used:25}\t'\ - .format(user=user, - key=key, - days=age_days, - access_key_last_used=_)) - key += 1 + self.print_key(user, key, active, age_days, last_used, created) + + # pylint: disable=too-many-arguments + def print_key(self, user, key, active, age_days, last_used, created): + log.debug('access key %s, active: %s, last used %s days ago > %s', key, active, age_days, self.age) + print('{user:20}\t{key}\t{active}\t{days:>3}\t{last_used:25}\t{created}'\ + .format(user=user, + key=key, + active='Active' if active else 'Inactive', + days=age_days, + last_used=last_used, + created=created + ) + ) if __name__ == '__main__': From 1cee9e5977c85848a0d323061c00abe6b43a1c87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 11:16:46 +0000 Subject: [PATCH 0150/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 17efe0c68..b7f45f612 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,8 @@ Environment variables are supported for convenience and also to hide credentials - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days + - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Hadoop](http://hadoop.apache.org/) & NoSQL: From debb9721fae59e57a204f4ee14dbb6a683b8d1d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 11:46:43 +0000 Subject: [PATCH 0151/2295] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 9d39caf33..c0026570d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,6 +24,8 @@ Status is Active or Inactive +Validated compared to xls report download from Trusted Advisor -> Security -> IAM Access Key Rotation + Uses the Boto library, read here for the list of ways to configure your AWS credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html From 60fa757168a15138dd1c4ce2be1cfc9f7f678ab0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 17:56:56 +0000 Subject: [PATCH 0152/2295] added aws_s3_presign_url.py --- aws_s3_presign_url.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 aws_s3_presign_url.py diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py new file mode 100755 index 000000000..c48a6147c --- /dev/null +++ b/aws_s3_presign_url.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Generate and print an S3 pre-signed URL + +Will generate a pre-signed URL even when the bucket and object key don't exist! + +(you will get a runtime error when requesting the link that the bucket or object doesn't exist) + + +Uses the Boto library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import boto3 + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +def main(): + parser = argparse.ArgumentParser( + description='Generate an AWS S3 pre-signed URL to access an S3 object without logging in') + parser.add_argument('bucket', help='Bucket Name') + parser.add_argument('key', help='Key') + parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds') + args = parser.parse_args() + + conn = boto3.client('s3') + url = conn.generate_presigned_url( + 'get_object', + Params={ + 'Bucket': args.bucket, + 'Key': args.key + }, + ExpiresIn=args.expiration + ) + print(url) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('Control-C...') From edace9cf841cedcb7203edca90b4e44c3a9b0b8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 20:33:36 +0000 Subject: [PATCH 0153/2295] updated aws_s3_presign_url.py --- aws_s3_presign_url.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py index c48a6147c..57f4d965d 100755 --- a/aws_s3_presign_url.py +++ b/aws_s3_presign_url.py @@ -35,10 +35,11 @@ from __future__ import unicode_literals import argparse +import sys import boto3 __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' def main(): parser = argparse.ArgumentParser( @@ -64,4 +65,4 @@ def main(): try: main() except KeyboardInterrupt: - print('Control-C...') + print('Control-C...', file=sys.stderr) From 4e498e905c2547236d7a2baef2b7f23d727b0749 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 10:20:49 +0000 Subject: [PATCH 0154/2295] updated aws_s3_presign_url.py --- aws_s3_presign_url.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py index 57f4d965d..2fbb1331f 100755 --- a/aws_s3_presign_url.py +++ b/aws_s3_presign_url.py @@ -18,6 +18,10 @@ Generate and print an S3 pre-signed URL +Can do the same with the following AWS CLI command: + +aws s3 presign s3:/// [--expires-in ] + Will generate a pre-signed URL even when the bucket and object key don't exist! (you will get a runtime error when requesting the link that the bucket or object doesn't exist) @@ -49,6 +53,7 @@ def main(): parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds') args = parser.parse_args() + # more useful if doing this programmatically as we can do this on the command line via AWS CLI conn = boto3.client('s3') url = conn.generate_presigned_url( 'get_object', From 990e1bb52bf53cab54e2b82d7ca11c6eadf08db5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 11:00:17 +0000 Subject: [PATCH 0155/2295] renamed aws_s3_presign_url.py to aws_s3_presign.py --- aws_s3_presign_url.py => aws_s3_presign.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename aws_s3_presign_url.py => aws_s3_presign.py (100%) diff --git a/aws_s3_presign_url.py b/aws_s3_presign.py similarity index 100% rename from aws_s3_presign_url.py rename to aws_s3_presign.py From dee9ea6ff2a2d9af8c1ca8f6be83e3268e860405 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:47:57 +0000 Subject: [PATCH 0156/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index b203357f4..3ab4db6a0 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -509,6 +509,9 @@ dest[137]="aws lambda update-function-code --function-name --zip-file src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' dest[138]=' aws elb create-load-balancer --load-balancer-name ...' +src[139]=' in column "blah" of table "blah2"' +dest[139]=' in column "" of table ""' + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 91ba1923014a7d235fa05aba0b97f68acac52657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:48:22 +0000 Subject: [PATCH 0157/2295] updated anonymize.py --- anonymize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 8210af2d6..06e6fa9c9 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.6' +__version__ = '0.10.7' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -317,6 +317,7 @@ def __init__(self): .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), + 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, @@ -451,6 +452,7 @@ def __init__(self): 'db': r'\1', 'db2': r'\1', 'db3': r'\1', + 'db4': r'\1<\2>', 'generic': r'\1://', 'generic2': r'\1', 'generic3': r'\1', From 9269ad9e12d8f331b1bf255bdadd8cf32e66bb82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:52:33 +0000 Subject: [PATCH 0158/2295] added --ignore-errors --- hive_tables_row_counts.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 80172c21a..39ca80814 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.3.0' +__version__ = '0.4.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -106,6 +106,10 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + # ignore tables that fail with errors like: + # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' + # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -176,8 +180,11 @@ def main(): continue try: get_row_counts(conn, args, database, table, partition_regex) - except impala.error.OperationalError as _: - log.error(_) + except Exception as _: + if args.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(conn, args, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From c997a0b2788c7f5943d263d6977d0e05ea95198d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:09:38 +0000 Subject: [PATCH 0159/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 39ca80814..58122be0d 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -106,9 +106,16 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - # ignore tables that fail with errors like: - # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' - # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# +# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# or this for Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From bbf4e0e18d7795b1402a41b267097841b745825d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:11:47 +0000 Subject: [PATCH 0160/2295] added --ignore-errors --- hive_foreach_table.py | 50 ++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index e52eca238..afe850bb2 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -68,7 +68,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -122,6 +122,17 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +# +# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# or this for Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -195,20 +206,29 @@ def main(): if _ == 'db': query = args.query.format(table=table) try: - log.info("running %s", query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise + execute(conn, database, table, query) + except Exception as _: + if args.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + +def execute(conn, database, table, query): + try: + log.info("running %s", query) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) + #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # log.error(_) + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': From 4ef8d1b24a614dc27c38f36298c8d5a64f366cc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:14:50 +0000 Subject: [PATCH 0161/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index afe850bb2..3e2163be3 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -215,7 +215,7 @@ def main(): def execute(conn, database, table, query): try: - log.info("running %s", query) + log.info(" %s.%s - running %s", database, table, query) with conn.cursor() as query_cursor: # doesn't support parameterized query quoting from dbapi spec query_cursor.execute(query) From 5ac651264dc633cf6f96f493e9e9cc24001ccc2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:29:53 +0000 Subject: [PATCH 0162/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 58122be0d..537afffc8 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -107,11 +107,13 @@ def parse_args(): help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # -# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): # # impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long # -# or this for Impala: +# Impala: # # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' From 7a88ad935bd3df2dda29055c091624c8c2eb41f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:30:46 +0000 Subject: [PATCH 0163/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 3e2163be3..90b294709 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -123,11 +123,13 @@ def parse_args(): help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # -# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): # # impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long # -# or this for Impala: +# Impala: # # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' From 9a93bcfd3c376532f40c481d274dc2eb769b0eb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:09:53 +0000 Subject: [PATCH 0164/2295] updated opentsdb_list_metrics.sh --- opentsdb_list_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh index 3a2675738..6365ffc35 100755 --- a/opentsdb_list_metrics.sh +++ b/opentsdb_list_metrics.sh @@ -72,7 +72,7 @@ done check_bin(){ local bin="$1" - if ! type -P $bin &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -83,6 +83,8 @@ check_bin jq if [ -z "${DEBUG:-}" ]; then curl_options="$curl_options -s" fi +# split opts +# shellcheck disable=SC2086 curl $curl_options "$tsd_url/api/suggest?type=$metrics&q=&max=2000000000" | jq '.[]' | sed 's/"//g' | From 9e1e67adfa395e8c09c9de2ca67751e1022581aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:11:43 +0000 Subject: [PATCH 0165/2295] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index 7308e1ab5..77a11f688 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_avro.py" @@ -26,8 +27,8 @@ section "Testing validate_avro.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_avro.py $@" - ./validate_avro.py $@ + echo "validate_avro.py $*" + ./validate_avro.py "$@" echo fi @@ -82,6 +83,7 @@ echo "testing stdin" ./validate_avro.py - < "$data_dir/test.avro" ./validate_avro.py < "$data_dir/test.avro" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_avro.py "$data_dir/test.avro" - < "$data_dir/test.avro" echo @@ -93,10 +95,10 @@ check_broken(){ ./validate_avro.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken avro in '$filename', returned exit code $exitcode" echo - #elif [ $exitcode != 0 ]; then + #elif [ "$exitcode" != 0 ]; then # echo "returned unexpected non-zero exit code $exitcode for broken avro in '$filename'" # exit 1 else From f0cc01acf341e9605afb141496d0f01727dc4037 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:52:10 +0000 Subject: [PATCH 0166/2295] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index 18822a584..ae415edc1 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_csv.py" @@ -26,8 +27,8 @@ section "Testing validate_csv.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_csv.py $@" - ./validate_csv.py $@ + echo "validate_csv.py $*" + ./validate_csv.py "$@" echo fi @@ -70,6 +71,7 @@ echo "testing stdin" ./validate_csv.py - < "$data_dir/test.csv" ./validate_csv.py < "$data_dir/test.csv" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_csv.py "$data_dir/test.csv" - < "$data_dir/test.csv" echo @@ -88,7 +90,7 @@ check_broken(){ ./validate_csv.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken csv in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -123,7 +125,7 @@ echo "checking blank content is invalid via stdin" check_broken - 2 < "$broken_dir/blank.csv" echo "checking blank content is invalid via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 +check_broken - 2 < /dev/null echo rm -fr "$broken_dir" From 5252e5906bec745975cb2bff08c4bbf286526b17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:53:49 +0000 Subject: [PATCH 0167/2295] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index 32a658e9f..d138aa22b 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_ini.py" @@ -26,8 +27,8 @@ section "Testing validate_ini.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_ini.py $@" - ./validate_ini.py $@ + echo "validate_ini.py $*" + ./validate_ini.py "$@" echo fi @@ -49,7 +50,8 @@ run_fail2(){ run_fail "${@/validate_ini/validate_ini2}" # ignore_run_unqualified } -if [ -f /etc/sssd/sssd.conf -a -r /etc/sssd/sssd.conf ]; then +if [ -f /etc/sssd/sssd.conf ] && + [ -r /etc/sssd/sssd.conf ]; then run ./validate_ini.py /etc/sssd/sssd.conf fi @@ -86,6 +88,7 @@ echo "testing stdin" run2 ./validate_ini.py - < "$data_dir/test.ini" run2 ./validate_ini.py < "$data_dir/test.ini" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 run2 ./validate_ini.py "$data_dir/test.ini" - < "$data_dir/test.ini" echo @@ -115,7 +118,7 @@ check_broken(){ ./validate_ini.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken ini in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -268,10 +271,10 @@ echo hr2 echo "checking blank content is invalid via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 -cat /dev/null | run_fail 2 ./validate_ini.py +check_broken - 2 < /dev/null +run_fail 2 ./validate_ini.py < /dev/null echo "validate_ini2.py blank content is valid:" -cat /dev/null | run ./validate_ini2.py +run ./validate_ini2.py < /dev/null echo hr2 From 5226ea26367dfac04e2023c3fc0c8a61961bde39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:56:56 +0000 Subject: [PATCH 0168/2295] updated test_validate_json.sh --- tests/test_validate_json.sh | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh index fb600bae2..279c7b5b9 100755 --- a/tests/test_validate_json.sh +++ b/tests/test_validate_json.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_json.py" @@ -26,8 +27,8 @@ section "Testing validate_json.py" export TIMEOUT=${TIMEOUT:-3} if [ $# -gt 0 ]; then - echo "validate_json.py $@" - ./validate_json.py $@ + echo "validate_json.py $*" + ./validate_json.py "$@" echo fi @@ -68,6 +69,7 @@ echo "testing stdin" ./validate_json.py - < "$data_dir/test.json" ./validate_json.py < "$data_dir/test.json" echo "testing stdin and file mix" +# shellcheck disable=SC2094 ./validate_json.py "$data_dir/test.json" - < "$data_dir/test.json" echo "testing stdin with multirecord" ./validate_json.py -m - < "$data_dir/multirecord.json" @@ -89,7 +91,7 @@ check_broken(){ ./validate_json.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken json in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -210,17 +212,21 @@ echo "checking --permit-single-quotes mode infers multirecord single quoted json echo echo "checking --permit-single-quotes mode works with multirecord single quoted json with mixed quoting (should result in a WARNING message)" -./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 | - grep -q WARNING && - echo "Found warning message" || - { echo "failed to raise a WARNING message for mixed quoting"; exit 1; } +if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 | grep -q WARNING; then + echo "Found warning message" +else + echo "failed to raise a WARNING message for mixed quoting" + exit 1 +fi echo echo "checking --permit-single-quotes mode infers multirecord single quoted json with mixed quoting (should result in a WARNING message)" -./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 | - grep -q WARNING && - echo "Found warning message" || - { echo "failed to raise a WARNING message for mixed quoting"; exit 1; } +if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 | grep -q WARNING; then + echo "Found warning message" +else + echo "failed to raise a WARNING message while inferring mixed quoting" + exit 1 +fi echo # ============================================================================ # @@ -309,7 +315,7 @@ echo "checking blank content is invalid for multirecord via stdin" check_broken - 2 -m < "$broken_dir/blank.json" echo "checking blank content is invalid for multirecord via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 -m +check_broken - 2 -m < /dev/null echo check_broken_sample_files json From f048bf472fabee35d37167116abfe171f33a3b7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:57:31 +0000 Subject: [PATCH 0169/2295] updated test_validate_ldap_ldif.sh --- tests/test_validate_ldap_ldif.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh index 566d90199..ab5a7654e 100755 --- a/tests/test_validate_ldap_ldif.sh +++ b/tests/test_validate_ldap_ldif.sh @@ -19,13 +19,14 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_ldap_ldif.py" if [ $# -gt 0 ]; then - echo "validate_ldap_ldif.py $@" - ./validate_ldap_ldif.py $@ + echo "validate_ldap_ldif.py $*" + ./validate_ldap_ldif.py "$@" echo fi @@ -60,6 +61,7 @@ echo echo "testing stdin" ./validate_ldap_ldif.py - < "$data_dir/add_ou.ldif" ./validate_ldap_ldif.py < "$data_dir/add_ou.ldif" +# shellcheck disable=SC2094 ./validate_ldap_ldif.py "$data_dir/add_ou.ldif" - < "$data_dir/add_ou.ldif" echo @@ -87,7 +89,7 @@ check_broken(){ ./validate_ldap_ldif.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken ldif in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From ef527c942b048eecb4895b54fcb68c2bd2f61864 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:58:00 +0000 Subject: [PATCH 0170/2295] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index 1efa5b4e6..9b5565419 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_multimedia.py" @@ -38,8 +39,8 @@ if ! type -P ffmpeg &>/dev/null; then fi if [ $# -gt 0 ]; then - echo "validate_multimedia.py $@" - ./validate_multimedia.py $@ + echo "validate_multimedia.py $*" + ./validate_multimedia.py "$@" echo fi @@ -87,7 +88,7 @@ check_broken(){ ./validate_multimedia.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken media in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From a36698204f21cc34bdaac7a21f5112445be04175 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:58:30 +0000 Subject: [PATCH 0171/2295] updated test_validate_parquet.sh --- tests/test_validate_parquet.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh index 01d8a0e0f..eb6e78e8b 100755 --- a/tests/test_validate_parquet.sh +++ b/tests/test_validate_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_parquet.py" @@ -32,8 +33,8 @@ fi #export TIMEOUT=10 if [ $# -gt 0 ]; then - echo "validate_parquet.py $@" - ./validate_parquet.py $@ + echo "validate_parquet.py $*" + ./validate_parquet.py "$@" echo fi @@ -84,6 +85,7 @@ echo "testing stdin" ./validate_parquet.py - < "$data_dir/test.parquet" ./validate_parquet.py < "$data_dir/test.parquet" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_parquet.py "$data_dir/test.parquet" - < "$data_dir/test.parquet" echo @@ -95,7 +97,7 @@ check_broken(){ ./validate_parquet.py -t 5 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken parquet in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From 8b4d4951ead31e3d0fe4000a1b763ce0d30f0897 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:59:09 +0000 Subject: [PATCH 0172/2295] updated test_validate_xml.sh --- tests/test_validate_xml.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh index ae6479497..5742faf45 100755 --- a/tests/test_validate_xml.sh +++ b/tests/test_validate_xml.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_xml.py" @@ -26,8 +27,8 @@ section "Testing validate_xml.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_xml.py $@" - ./validate_xml.py $@ + echo "validate_xml.py $*" + ./validate_xml.py "$@" echo fi @@ -60,6 +61,7 @@ echo echo "testing stdin" ./validate_xml.py - < "$data_dir/simple.xml" ./validate_xml.py < "$data_dir/simple.xml" +# shellcheck disable=SC2094 ./validate_xml.py "$data_dir/simple.xml" - < "$data_dir/simple.xml" echo @@ -77,7 +79,7 @@ check_broken(){ ./validate_xml.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken xml in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From 6d51e104f939ceb2dcb70907583cba587b15c167 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:59:46 +0000 Subject: [PATCH 0173/2295] updated test_validate_yaml.sh --- tests/test_validate_yaml.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index 7ab5b2975..d8b0e265c 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_yaml.py" @@ -29,8 +30,8 @@ if is_inside_docker; then fi if [ $# -gt 0 ]; then - echo "validate_yaml.py $@" - ./validate_yaml.py $@ + echo "validate_yaml.py $*" + ./validate_yaml.py "$@" echo fi @@ -65,6 +66,7 @@ echo "testing stdin" ./validate_yaml.py - < "$data_dir/test.yaml" ./validate_yaml.py < "$data_dir/test.yaml" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_yaml.py "$data_dir/test.yaml" - < "$data_dir/test.yaml" echo @@ -82,7 +84,7 @@ check_broken(){ ./validate_yaml.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken yaml in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From f4cdfb53a2e7c5149b45096adfff722b9b55573f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 15:01:33 +0000 Subject: [PATCH 0174/2295] updated opentsdb_list_metrics_hbase.sh --- opentsdb_list_metrics_hbase.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh index 56aaddfa9..945b108ff 100755 --- a/opentsdb_list_metrics_hbase.sh +++ b/opentsdb_list_metrics_hbase.sh @@ -73,7 +73,7 @@ done check_bin(){ local bin="$1" - if ! type -P $bin &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -106,6 +106,7 @@ for line in sys.stdin: # ts = metrics[metric] print("{}\t{}\t{}".format(ts, time.strftime("%F %T", time.localtime(int(ts)/1000)), metric)) EOF + # shellcheck disable=SC2064 trap "rm '$tmp_python_script'" EXIT hbase shell <<< "scan 'tsdb-uid', { COLUMNS => 'name:$metrics', VERSIONS => 1 }" 2>/dev/null | From 771824123ff5cb0fa6cad022ecff7067c92ef0bf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 17:58:09 +0000 Subject: [PATCH 0175/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 537afffc8..d4d685f26 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -118,7 +118,7 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -190,6 +190,8 @@ def main(): try: get_row_counts(conn, args, database, table, partition_regex) except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out if args.ignore_errors: log.error("database '%s' table '%s': %s", database, table, _) continue From 010dfb90db60aef2c9984a8afd572a08e1cb82af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:29:12 +0000 Subject: [PATCH 0176/2295] updated all.sh --- tests/all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/all.sh b/tests/all.sh index fabd97170..18dddfc25 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -17,6 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck disable=SC1090 . "$srcdir/utils.sh" # imported by utils.sh above From cdbf33b67cd8f792f070d9a453f11ea9e6c4728f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:29:36 +0000 Subject: [PATCH 0177/2295] updated check.sh --- tests/check.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index 4541f684b..07e1bd9c2 100755 --- a/tests/check.sh +++ b/tests/check.sh @@ -21,12 +21,12 @@ check(){ msg=$2 echo hr2 - echo $msg + echo "$msg" hr2 echo - echo cmd: $cmd + echo "cmd: $cmd" echo - if eval $cmd; then + if eval "$cmd"; then echo echo "SUCCESS" else From 01671a19207886fe96a42c35146293488e67a79c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:46:55 +0000 Subject: [PATCH 0178/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 3ab4db6a0..829cf04b0 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -18,7 +18,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -test_nums="${@:-}" +test_nums="${*:-}" parallel="" if [ "$test_nums" = "p" ]; then parallel="1" @@ -27,6 +27,7 @@ fi cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Anonymize" @@ -45,7 +46,7 @@ if [ -z "$test_nums" ]; then echo echo "checking file args:" run++ - if [ `$anonymize -ae README.md | wc -l` -gt 100 ]; then + if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then echo "SUCCEEDED - anonymized README.md > 100 lines" else echo "FAILED - suspicious README.md file arg result came to <= 100 lines" @@ -506,6 +507,7 @@ dest[136]="arn:aws:acm:us-east-1::function::7" src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" +# shellcheck disable=SC2016 src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' dest[138]=' aws elb create-load-balancer --load-balancer-name ...' @@ -523,12 +525,16 @@ dest[139]=' in column "" of table "
"' args="-aPe" test_anonymize(){ run++ - src="$1" - dest="$2" + # shellcheck disable=SC2178 + local src="$1" + # shellcheck disable=SC2178 + local dest="$2" #[ -z "${src[$i]:-}" ] && { echo "skipping test $i..."; continue; } # didn't work for \e escape codes for ANSI stripping test #result="$(echo -e "$src" | $anonymize $args)" + # shellcheck disable=SC2128 result="$($anonymize $args <<< "$src")" + # shellcheck disable=SC2128 if grep -xFq -- "$dest" <<< "$result"; then echo -n "SUCCEEDED anonymization test $i" if [ -n "${SHOW_OUTPUT:-}" ]; then @@ -560,7 +566,7 @@ fi # this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle #for (( i = 0 ; i < ${#src[@]} ; i++ )); do run_tests(){ - test_numbers="${@:-${!src[@]}}" + test_numbers="${*:-${!src[*]}}" for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } [ -n "${dest[$i]:-}" ] || { echo "code error: dest[$i] not defined"; exit 1; } @@ -614,6 +620,8 @@ if [ -n "$parallel" ]; then fi echo +# run_count assigned in utils lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "SUCCESS! All tests for $anonymize completed in" echo From a57afb0d1591baa0a030b206b54bbcde9761ecdb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 14:09:11 +0000 Subject: [PATCH 0179/2295] updated python3.sh --- tests/python3.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python3.sh b/tests/python3.sh index eeaf5f9f3..9cd102ac1 100755 --- a/tests/python3.sh +++ b/tests/python3.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh pip install caniusepython3 From 006cc4e89ccbdd188d6b96e5b1135a5f62ab16a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 14:11:05 +0000 Subject: [PATCH 0180/2295] updated syntax.sh --- tests/syntax.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/syntax.sh b/tests/syntax.sh index 26a8f16c8..ea6b33a6a 100755 --- a/tests/syntax.sh +++ b/tests/syntax.sh @@ -19,20 +19,21 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh -for x in $(echo *.py *.jy 2>/dev/null); do - isExcluded "$x" && continue +while read -r prog; do + isExcluded "$prog" && continue if type -P flake8 &> /dev/null; then - echo "flake8 $x" - flake8 --max-line-length=120 --statistics $x + echo "flake8 $prog" + flake8 --max-line-length=120 --statistics "$prog" echo; hr; echo fi for y in pyflakes pychecker; do - if type -P $y &>/dev/null; then - echo "$y $x" - $y $x + if type -P "$y" &>/dev/null; then + echo "$y $prog" + "$y" "$prog" echo; hr; echo fi done -done +done < <(find . -type f -name '*.py' -o -type f -name '*.jy') From b8f6d77b752dbb718c2eb4ebde40cf66c5cc7171 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 15:18:49 +0000 Subject: [PATCH 0181/2295] updated test_apache-drill.sh --- tests/test_apache-drill.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index 03c6539a8..c9a675207 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -20,11 +20,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1090 . "$srcdir/utils.sh" section "A p a c h e D r i l l" -export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}" +export APACHE_DRILL_VERSIONS="${*:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}" APACHE_DRILL_HOST="${DOCKER_HOST:-${APACHE_DRILL_HOST:-${HOST:-localhost}}}" APACHE_DRILL_HOST="${APACHE_DRILL_HOST##*/}" @@ -45,6 +46,7 @@ test_apache_drill(){ echo "getting Apache Drill dynamic port mappings:" docker_compose_port "Apache Drill" hr + # shellcheck disable=SC2153 when_ports_available "$APACHE_DRILL_HOST" "$APACHE_DRILL_PORT" hr when_url_content "http://$APACHE_DRILL_HOST:$APACHE_DRILL_PORT/status" "Running" @@ -60,15 +62,19 @@ test_apache_drill(){ hr APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill.py $non_drill_node1 $non_drill_node2 + # shellcheck disable=SC2097,SC2098 APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT" # Drill 1.7+ only - if [ "$version" = "latest" ] || [[ "$version" > 1.6 ]]; then + if [ "$version" = "latest" ] || [ "$(bc <<< "$version > 1.6")" = 1 ]; then APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill2.py $non_drill_node1 $non_drill_node2 + # shellcheck disable=SC2097,SC2098 APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill2.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT" fi + # run_count defined in util lib + # shellcheck disable=SC2154 echo "Completed $run_count Apache Drill tests" hr [ -n "${KEEPDOCKER:-}" ] || From 6d417263c0d2ac21e50759915489ca47c6aa2e84 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 21 Jan 2020 14:54:43 +0000 Subject: [PATCH 0182/2295] moved STS token to evaluate before secret key --- anonymize.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/anonymize.py b/anonymize.py index 06e6fa9c9..07bc730ac 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.7' +__version__ = '0.10.8' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -300,8 +300,8 @@ def __init__(self): 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/.-]+', # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html 'aws3': r'\bAKIA[A-Za-z0-9]{16}\b', # access key - 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key - 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key 'aws7': r'\bsg-[A-Za-z0-9]{8}(?\2<\3>', 'aws2': r'\1:', 'aws3': r'', - 'aws4': r'', - 'aws5': r'', + 'aws4': r'', + 'aws5': r'', 'aws6': r'', 'aws7': r'', 'aws8': r'\1:///', From 3574d27cf57070c79bbd617b48f0b0312a5e50d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:15:37 +0000 Subject: [PATCH 0183/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 384e5f8b3..a134b3ec8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 384e5f8b366fdeb6a73363dbf388824f5de5e7bc +Subproject commit a134b3ec87c26561ee43180be67b662665a72542 From 70395a5e784481b03c185e99009215b9f699bbb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:15:38 +0000 Subject: [PATCH 0184/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f3fb9b2a9..9cf83f21c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f3fb9b2a9dedb8a4ed7e85183f1739ad2a63fd92 +Subproject commit 9cf83f21c39fdb250d4e8a155d08220b204adaee From 48ec6ca7d745d639253b9f9059dfbde1fd7965a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:23 +0000 Subject: [PATCH 0185/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a134b3ec8..1e33eaec4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a134b3ec87c26561ee43180be67b662665a72542 +Subproject commit 1e33eaec4117818ee32e3644e101a715f5a66de0 From 8b8411cc017d44bd3acb0708d41657a16691fdaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:24 +0000 Subject: [PATCH 0186/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9cf83f21c..4dcc62e62 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9cf83f21c39fdb250d4e8a155d08220b204adaee +Subproject commit 4dcc62e62cbc569a0422a1f57be20ceef32054d0 From d5ca483db87946ac187209d168965c2d541a5988 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:42 +0000 Subject: [PATCH 0187/2295] updated README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b7f45f612..cc620fcad 100644 --- a/README.md +++ b/README.md @@ -336,12 +336,13 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (Hive, Impala, MySQL, PostgreSQL, Cassandra CQL, Apache Drill, Couchbase N1QL, Microsoft SQL Server, Oracle, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc. +* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... + * [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. * [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 From e3edbb228461ea458a9eafb07b35ba5b59ca51e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:58:34 +0000 Subject: [PATCH 0188/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cc620fcad..114ee40ca 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon +###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 6822bed726e7bbbfa251e980496c107482e0454b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 11:51:32 +0000 Subject: [PATCH 0189/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 114ee40ca..c32604247 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -https://www.linkedin.com/in/harisekhon +[https://www.linkedin.com/in/harisekhon](https://www.linkedin.com/in/harisekhon) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 112337e86d8bcde11bf63f5735322ecf6aecc82a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:10 +0000 Subject: [PATCH 0190/2295] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index 77a11f688..e7c595d51 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -90,7 +90,7 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_avro.py -t 1 $options "$filename" exitcode=$? From b2777b7efbfd0a014c04d181c1e328e8c5dac63e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:25 +0000 Subject: [PATCH 0191/2295] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index ae415edc1..ac9813a03 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -85,7 +85,7 @@ hr2 check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_csv.py -t 1 $options "$filename" exitcode=$? From 96f3e864c7f2bbef8ff68a32bb95627179e938e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:43 +0000 Subject: [PATCH 0192/2295] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index d138aa22b..bcfaebfd8 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -113,7 +113,7 @@ export TIMEOUT=1 check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_ini.py $options "$filename" exitcode=$? From d2bcdba4378d65cd583b59034640b967ebc540e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:21:21 +0000 Subject: [PATCH 0193/2295] updated test_validate_json.sh --- tests/test_validate_json.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh index 279c7b5b9..39b24306b 100755 --- a/tests/test_validate_json.sh +++ b/tests/test_validate_json.sh @@ -86,8 +86,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_json.py $options "$filename" exitcode=$? set -e From 89149c79ea6ff6bcbbd64c42334ec069c8d48eb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:21:46 +0000 Subject: [PATCH 0194/2295] updated test_validate_ldap_ldif.sh --- tests/test_validate_ldap_ldif.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh index ab5a7654e..53af23074 100755 --- a/tests/test_validate_ldap_ldif.sh +++ b/tests/test_validate_ldap_ldif.sh @@ -84,8 +84,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_ldap_ldif.py $options "$filename" exitcode=$? set -e From 494164e76d0681e447eb211567183316b1550cd0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:22:07 +0000 Subject: [PATCH 0195/2295] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index 9b5565419..7302ca55d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -83,8 +83,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_multimedia.py -t 1 $options "$filename" exitcode=$? set -e From 8a843376d8632c0818c91b49185c5d8b5aa66abc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:22:35 +0000 Subject: [PATCH 0196/2295] updated test_validate_parquet.sh --- tests/test_validate_parquet.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh index eb6e78e8b..d75942e84 100755 --- a/tests/test_validate_parquet.sh +++ b/tests/test_validate_parquet.sh @@ -92,8 +92,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_parquet.py -t 5 $options "$filename" exitcode=$? set -e From b64eba0958f2f966c2e9daa92200a703798ec430 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:04 +0000 Subject: [PATCH 0197/2295] updated test_validate_xml.sh --- tests/test_validate_xml.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh index 5742faf45..37b1ee106 100755 --- a/tests/test_validate_xml.sh +++ b/tests/test_validate_xml.sh @@ -74,8 +74,9 @@ echo "Now trying non-xml files to detect successful failure:" check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_xml.py -t 1 $options "$filename" exitcode=$? set -e From 5091b540113a208d52428b3d5bbf829cbea6b3ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:26 +0000 Subject: [PATCH 0198/2295] updated test_validate_yaml.sh --- tests/test_validate_yaml.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index d8b0e265c..098cf827d 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -79,8 +79,9 @@ echo "Now trying non-yaml files to detect successful failure:" check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_yaml.py -t 1 $options "$filename" exitcode=$? set -e From a8a0bacce82f8bc0372d8531fc28ab7fb7e81be8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:46 +0000 Subject: [PATCH 0199/2295] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index e7c595d51..d8dab1d88 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -92,6 +92,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_avro.py -t 1 $options "$filename" exitcode=$? set -e From b7e82109e1439b4a2c3c75956f50167d5c321b72 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:24:03 +0000 Subject: [PATCH 0200/2295] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index ac9813a03..680b63390 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -87,6 +87,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_csv.py -t 1 $options "$filename" exitcode=$? set -e From ec37e03857d344b85dd85d8abde8816c2b5ea624 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:24:14 +0000 Subject: [PATCH 0201/2295] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index bcfaebfd8..f2282be6a 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -115,6 +115,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_ini.py $options "$filename" exitcode=$? set -e From 02ec92023ace876cf5ccd35d88cb9f1ef16d14fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:31:53 +0000 Subject: [PATCH 0202/2295] updated test_timeout.sh --- tests/test_timeout.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh index cadfde058..4ce6753d2 100755 --- a/tests/test_timeout.sh +++ b/tests/test_timeout.sh @@ -15,7 +15,6 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x - srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$srcdir/../timeout.py -t 2 sleep 10 || : +"$srcdir/../timeout.py" -t 2 sleep 10 || : From 0917b820ac11d2a96aa1ce9bf23550f45819da48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:33:19 +0000 Subject: [PATCH 0203/2295] updated test_timeout.sh --- tests/test_timeout.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh index 4ce6753d2..26d38d589 100755 --- a/tests/test_timeout.sh +++ b/tests/test_timeout.sh @@ -17,4 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -"$srcdir/../timeout.py" -t 2 sleep 10 || : +# shellcheck disable=SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" + +run_fail 3 "$srcdir/../timeout.py" -t 2 sleep 10 From b3117a12868df50da8a185de1b4ee542bb0eeb75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:34:50 +0000 Subject: [PATCH 0204/2295] updated test_strip_ansi_escape_codes.sh --- tests/test_strip_ansi_escape_codes.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_strip_ansi_escape_codes.sh b/tests/test_strip_ansi_escape_codes.sh index b885c1878..d0effe274 100755 --- a/tests/test_strip_ansi_escape_codes.sh +++ b/tests/test_strip_ansi_escape_codes.sh @@ -15,6 +15,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Strip ANSI Escape Codes" @@ -23,11 +24,11 @@ name="strip_ansi_escape_codes.py" start_time=$(date +%s) -if is_mac; then - cat_opts="-e" -else - cat_opts="-A" -fi +#if is_mac; then +# cat_opts="-e" +#else +# cat_opts="-A" +#fi run++ if echo "some highlighted content" | grep --color=yes highlighted | @@ -42,6 +43,7 @@ fi hr tmp=$(mktemp /tmp/strip_ansi_escape_codes.XXXXX) +# shellcheck disable=SC2064,SC2086 trap "rm $tmp" $TRAP_SIGNALS echo @@ -62,6 +64,8 @@ tee /dev/stderr | fi echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "All version tests for $name completed in" echo From 0c541e32bf4914f6ff9fb22eb1aab28d16bb8a8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:36:19 +0000 Subject: [PATCH 0205/2295] updated test_center.sh --- tests/test_center.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_center.sh b/tests/test_center.sh index 976466e1a..0dfd434f8 100755 --- a/tests/test_center.sh +++ b/tests/test_center.sh @@ -22,6 +22,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing center.py" @@ -127,6 +128,8 @@ echo "testing spacing with stdin:" run_output "$expected" ./center.py -s <<< " " echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Completed $run_count tests" echo echo "All tests for center.py completed successfully" From e8a2007fcc5c8450a8fabb345ae1f038d4530db7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:39:54 +0000 Subject: [PATCH 0206/2295] updated welcome.py --- welcome.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/welcome.py b/welcome.py index 97f6f173f..0d93b836f 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.1' +__version__ = '2.0.2' class Welcome(CLI): @@ -56,6 +56,7 @@ def __init__(self): # Python 3.x # super().__init__() self.quick = False + self.timeout_default = 20 @staticmethod def case_user(user): From c2118940c23924ebe7d5c12ff7d207fc69523087 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:40:38 +0000 Subject: [PATCH 0207/2295] updated test_docker.sh --- tests/test_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_docker.sh b/tests/test_docker.sh index ddb02baa7..8d00d3be3 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/docker.sh" + +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Docker Image" From 10024bd385eb8055dec70de3bab07a667dcbd1b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:52:09 +0000 Subject: [PATCH 0208/2295] updated test_docker_registry_show_tags.sh --- tests/test_docker_registry_show_tags.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh index d3d72b932..bc0612399 100755 --- a/tests/test_docker_registry_show_tags.sh +++ b/tests/test_docker_registry_show_tags.sh @@ -19,8 +19,10 @@ srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir2/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" srcdir="$srcdir2" @@ -75,12 +77,13 @@ private_key="registry.key" certificate="registry.crt" htpasswd="registry.htpasswd" csr="registry.csr" -if ! [ -f "$private_key" -a -f "$certificate" ]; then +if ! [ -f "$private_key" ] && + [ -f "$certificate" ]; then echo "Generating sample SSL certificates:" echo openssl genrsa -out "$private_key" 2048 echo - yes "" | openssl req -new -key "$private_key" -out "$csr" || : + yes "." | openssl req -new -key "$private_key" -out "$csr" || : echo openssl x509 -req -days 3650 -in "$csr" -signkey "$private_key" -out "$certificate" echo @@ -108,6 +111,7 @@ echo "getting dynamic Docker Registry port mapping:" docker_compose_port "Docker Registry" hr +# shellcheck disable=SC2153 if [ -z "$DOCKER_REGISTRY_PORT" ]; then echo "DOCKER_REGISTRY_PORT not found from running container, did container fail to start up properly?" exit 1 From b2347e4e2eb4df7862717122ffcaa4e7e2f5fc4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:53:12 +0000 Subject: [PATCH 0209/2295] updated test_dockerfiles_check_git_branches.sh --- tests/test_dockerfiles_check_git_branches.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh index ec2be2a11..2529a2c9c 100755 --- a/tests/test_dockerfiles_check_git_branches.sh +++ b/tests/test_dockerfiles_check_git_branches.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git branches" From 9f8b74267ee327995b204fe5f4d83ae9c2b9c442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:53:22 +0000 Subject: [PATCH 0210/2295] updated test_dockerfiles_check_git_tags.sh --- tests/test_dockerfiles_check_git_tags.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh index 33556738c..ada872f3c 100755 --- a/tests/test_dockerfiles_check_git_tags.sh +++ b/tests/test_dockerfiles_check_git_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git Tags" From 8fb94d8288a7f4f45d62a8cb38a3a86342e4e29c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:54:05 +0000 Subject: [PATCH 0211/2295] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 73b8bf7b5..ee82c5bb8 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" @@ -33,6 +35,7 @@ check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED +# shellcheck disable=SC2016 check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" echo From b0e31398e31366d2d59042d5a31a4547faa35c5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:54:27 +0000 Subject: [PATCH 0212/2295] updated test_dockerhub_show_tags.sh --- tests/test_dockerhub_show_tags.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh index 71f424a3f..8d052f174 100755 --- a/tests/test_dockerhub_show_tags.sh +++ b/tests/test_dockerhub_show_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" @@ -39,6 +41,8 @@ echo echo echo "All DockerHub Show Tags tests completed successfully" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "DockerHub Show Tags tests completed in" echo From a4f4d50064a2a4e4a449e5cfa5be7eb967f6efb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:55:55 +0000 Subject: [PATCH 0213/2295] updated test_elasticsearch.sh --- tests/test_elasticsearch.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_elasticsearch.sh b/tests/test_elasticsearch.sh index e18fd8d09..5abd6e53f 100755 --- a/tests/test_elasticsearch.sh +++ b/tests/test_elasticsearch.sh @@ -19,11 +19,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "E l a s t i c s e a r c h" -export ELASTICSEARCH_VERSIONS="${@:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}" +export ELASTICSEARCH_VERSIONS="${*:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}" ELASTICSEARCH_HOST="${DOCKER_HOST:-${ELASTICSEARCH_HOST:-${HOST:-localhost}}}" ELASTICSEARCH_HOST="${ELASTICSEARCH_HOST##*/}" @@ -43,7 +44,7 @@ test_elasticsearch(){ local version="$1" section2 "Setting up Elasticsearch $version test container" if [ "$version" != "latest" ] && [ "${version:0:1}" -ge 6 ]; then - local export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml" + export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml" fi docker_compose_pull VERSION="$version" docker-compose up -d @@ -63,6 +64,7 @@ test_elasticsearch(){ ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2 + # shellcheck disable=SC2097,SC2098 ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \ run_grep "^$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2 "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT" From 0803c70901d57707ae36b71d3b0cab1d5764f3dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:58:00 +0000 Subject: [PATCH 0214/2295] updated test_find_active_server.sh --- tests/test_find_active_server.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 74bb8e494..1bde9e0e6 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . bash-tools/lib/utils.sh set +e +o pipefail @@ -174,19 +175,19 @@ count_socket_attempts=0 found_google_socket=0 found_duckduckgo_socket=0 run++ -for x in {1..10}; do +for _ in {1..10}; do echo -n . - let count_socket_attempts+=1 + ((count_socket_attempts+=1)) output="$(./find_active_server.py -n1 --random --port 80 $WEBSITE1 $WEBSITE2)" if [ "$output" = "$WEBSITE2" ]; then found_google_socket=1 elif [ "$output" = "$WEBSITE1" ]; then found_duckduckgo_socket=1 fi - [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ] && break + [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ] && break done echo -if [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ]; then +if [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ]; then echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs" else die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs" @@ -210,25 +211,27 @@ count_http_attempts=0 found_google_http=0 found_duckduckgo_http=0 run++ -for x in {1..10}; do +for _ in {1..10}; do echo -n . - let count_http_attempts+=1 + ((count_http_attempts+=1)) output="$(./find_active_server.py -n1 --http --random $WEBSITE1 $WEBSITE2)" if [ "$output" = "$WEBSITE2" ]; then found_google_http=1 elif [ "$output" = "$WEBSITE1" ]; then found_duckduckgo_http=1 fi - [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ] && break + [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ] && break done echo -if [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ]; then +if [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ]; then echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs" else die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs" fi hr echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_active_server.py tests completed in" echo From cc18f5690ac8b7e8d67e24eaaefc3de6c25a08a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:59:05 +0000 Subject: [PATCH 0215/2295] updated test_find_duplicate_files.sh --- tests/test_find_duplicate_files.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh index 807508f52..9a4e951c9 100755 --- a/tests/test_find_duplicate_files.sh +++ b/tests/test_find_duplicate_files.sh @@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "find_duplicate_files.py" @@ -29,6 +30,7 @@ start_time="$(start_timer "find_duplicate_files.py test")" testdir1="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files.XXXXXX)" testdir2="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files2.XXXXXX)" +# shellcheck disable=SC2064,SC2086 trap "rm -fr '$testdir1' '$testdir2'" $TRAP_SIGNALS echo test > "$testdir1/test1.txt" @@ -135,6 +137,8 @@ rm -fr "$testdir1" "$testdir2" echo echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_duplicate_files.py tests completed in" echo From 229de7ed0b9541ec0c918afbc02d98ae8dd7f1f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:00:03 +0000 Subject: [PATCH 0216/2295] updated test_getent.sh --- tests/test_getent.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_getent.sh b/tests/test_getent.sh index d2b87ddfe..26a7aa2b5 100755 --- a/tests/test_getent.sh +++ b/tests/test_getent.sh @@ -19,34 +19,38 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . ./bash-tools/lib/utils.sh section "Getent" start_time="$(start_timer "find_active_server.py test")" -system=`uname -s` +system="$(uname -s)" echo "system = $system" hr -if [ "$system" = "Linux" -o "$system" = Darwin ]; then +if [ "$system" = "Linux" ] || + [ "$system" = Darwin ]; then run ./getent.py passwd | grep -v ':[x*!]*:' # counter is lost in subshell, increment manually run++ # $USER isn't always available in docker containers, use 'id' instead - run ./getent.py passwd `id -un` + run ./getent.py passwd "$(id -un)" run_fail 2 ./getent.py passwd nonexistentuser run ./getent.py group | grep -v ':[x*!]:' run++ - run ./getent.py group `id -gn` + run ./getent.py group "$(id -gn)" run_fail 2 ./getent.py group nonexistentgroup echo + # $run_count defined in lib + # shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_active_server.py tests completed in" else From ab829ff61b39faa1075a3aa7aaa595ab8333ad0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:01:23 +0000 Subject: [PATCH 0217/2295] updated test_hadoop.sh --- tests/test_hadoop.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_hadoop.sh b/tests/test_hadoop.sh index 25b705f8a..c2ebedfe9 100755 --- a/tests/test_hadoop.sh +++ b/tests/test_hadoop.sh @@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1090 . "$srcdir/utils.sh" section "H a d o o p" # find_active_hadoop_namenode.py doesn't work on Hadoop 2.2 as the JMX bean isn't present -export HADOOP_VERSIONS="${@:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}" +export HADOOP_VERSIONS="${*:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}" HADOOP_HOST="${DOCKER_HOST:-${HADOOP_HOST:-${HOST:-localhost}}}" HADOOP_HOST="${HADOOP_HOST##*/}" @@ -61,6 +62,7 @@ test_hadoop(){ docker_compose_port HADOOP_YARN_NODE_MANAGER_PORT "Yarn NM" export HADOOP_PORTS="$HADOOP_NAMENODE_PORT $HADOOP_DATANODE_PORT $HADOOP_YARN_RESOURCE_MANAGER_PORT $HADOOP_YARN_NODE_MANAGER_PORT" hr + # shellcheck disable=SC2086 when_ports_available "$HADOOP_HOST" $HADOOP_PORTS hr # don't use the worker nodes so not testing for their availability @@ -125,10 +127,12 @@ EOFCOMMENTED # therefore reset the HADOOP PORTS to point to something that should get connection refused like port 1 and so that the failure hosts still fail and return only the expected correct host HADOOP_NAMENODE_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_namenode.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_DATANODE_PORT" + # shellcheck disable=SC2097,SC2098 HADOOP_NAMENODE_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_NAMENODE_PORT$" ./find_active_hadoop_namenode.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_DATANODE_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_NAMENODE_PORT" HADOOP_YARN_RESOURCE_MANAGER_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT" + # shellcheck disable=SC2097,SC2098 HADOOP_YARN_RESOURCE_MANAGER_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT" [ -z "${KEEPDOCKER:-}" ] || docker-compose down From 89d1389af075fa9ba5d1ebd3486cf48cef98dc9d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:06:26 +0000 Subject: [PATCH 0218/2295] updated test_hbase.sh --- tests/test_hbase.sh | 93 ++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index ebde33a96..28186c242 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -16,14 +16,15 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$srcdir2/.." +cd "$srcdir/.." -. "$srcdir2/utils.sh" -. "$srcdir2/../bash-tools/lib/docker.sh" +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" -srcdir="$srcdir2" +# shellcheck disable=SC1090 +. "$srcdir/../bash-tools/lib/docker.sh" section "H B a s e" @@ -39,7 +40,7 @@ export HBASE_THRIFT_PORT_DEFAULT=9090 export HBASE_THRIFT_UI_PORT_DEFAULT=9095 export ZOOKEEPER_PORT_DEFAULT=2181 -export HBASE_VERSIONS="${@:-latest 0.96 0.98 1.0 1.1 1.2 1.3}" +export HBASE_VERSIONS="${*:-latest 0.96 0.98 1.0 1.1 1.2 1.3}" check_docker_available @@ -59,9 +60,10 @@ test_hbase(){ fi VERSION="$version" docker-compose up -d hr - if [ "$version" = "0.96" -o "$version" = "0.98" ]; then - local export HBASE_MASTER_PORT_DEFAULT=60010 - local export HBASE_REGIONSERVER_PORT_DEFAULT=60301 + if [ "$version" = "0.96" ] || + [ "$version" = "0.98" ]; then + export HBASE_MASTER_PORT_DEFAULT=60010 + export HBASE_REGIONSERVER_PORT_DEFAULT=60301 fi echo "getting HBase dynamic port mappings:" docker_compose_port "HBase Master" @@ -73,6 +75,7 @@ test_hbase(){ #docker_compose_port ZOOKEEPER_PORT "HBase ZooKeeper" export HBASE_PORTS="$HBASE_MASTER_PORT $HBASE_REGIONSERVER_PORT $HBASE_STARGATE_PORT $HBASE_STARGATE_UI_PORT $HBASE_THRIFT_PORT $HBASE_THRIFT_UI_PORT" hr + # shellcheck disable=SC2086 when_ports_available "$HBASE_HOST" $HBASE_PORTS hr if [ "${version:0:3}" = "0.9" ]; then @@ -118,10 +121,12 @@ EOF return fi # will otherwise pick up HBASE_HOST and use default port and return the real HBase Master + # shellcheck disable=SC2097,SC2098 HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hbase_master.py 127.0.0.2 127.0.0.3 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT" # if HBASE_PORT / --port is set to same as suffix then only outputs host not host:port + # shellcheck disable=SC2097,SC2098 HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \ run_grep "^$HBASE_HOST:$HBASE_MASTER_PORT$" ./find_active_hbase_master.py 127.0.0.2 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT" 127.0.0.3 "$HBASE_HOST:$HBASE_MASTER_PORT" @@ -245,53 +250,53 @@ EOF run_conn_refused ./hbase_table_row_key_distribution.py -T HexStringSplitTable # ============================================================================ # - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average --skip-zeros + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --average + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --skip-zeros + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --average - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST --count 2 --interval 2 + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" --count 2 --interval 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST --count 2 -i 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 --average + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" --count 2 -i 2 + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2 --average # ============================================================================ # - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --average + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --average - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 -T read,write,total - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --type read,write,total --average + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 -T read,write,total + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --type read,write,total --average - run ./hbase_regionserver_requests.py $HBASE_HOST --count 2 --interval 2 + run ./hbase_regionserver_requests.py "$HBASE_HOST" --count 2 --interval 2 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST --count 2 -i 2 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1 --average + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1 + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" --count 2 -i 2 + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1 --average # ============================================================================ # - run ./hbase_regions_by_size.py $HBASE_HOST - run ./hbase_regions_by_size.py $HBASE_HOST --smallest - run ./hbase_regions_by_size.py $HBASE_HOST --human - run ./hbase_regions_by_size.py $HBASE_HOST --human -s - run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10 - run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10 --smallest - - run ./hbase_regions_by_memstore_size.py $HBASE_HOST - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --smallest - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human -s - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10 - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10 --smallest + run ./hbase_regions_by_size.py "$HBASE_HOST" + run ./hbase_regions_by_size.py "$HBASE_HOST" --smallest + run ./hbase_regions_by_size.py "$HBASE_HOST" --human + run ./hbase_regions_by_size.py "$HBASE_HOST" --human -s + run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10 + run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10 --smallest + + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --smallest + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human -s + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10 + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10 --smallest # ============================================================================ # - run ./hbase_regions_least_used.py $HBASE_HOST -r 20000 - run ./hbase_regions_least_used.py $HBASE_HOST -r 0 - run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000 - run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000 --top 10 + run ./hbase_regions_least_used.py "$HBASE_HOST" -r 20000 + run ./hbase_regions_least_used.py "$HBASE_HOST" -r 0 + run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000 + run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000 --top 10 [ -z "${KEEPDOCKER:-}" ] || docker-compose down From 1d0ff74fa9561121e2acf0a451cd77402d0a901f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:07:13 +0000 Subject: [PATCH 0219/2295] updated test_hexanonymize.sh --- tests/test_hexanonymize.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_hexanonymize.sh b/tests/test_hexanonymize.sh index 57da1e297..9477001b2 100755 --- a/tests/test_hexanonymize.sh +++ b/tests/test_hexanonymize.sh @@ -15,6 +15,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "HexAnonymize" @@ -34,7 +35,9 @@ run++ check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" -time_taken "$start_time" "All version tests for $name completed in" +time_taken "$start_time" "All version tests for hexanonymize.py completed in" echo untrap From 78f5b4bdd1e44edb836642ca258a9079e9d8a76c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:07:55 +0000 Subject: [PATCH 0220/2295] updated test_headtail.sh --- tests/test_headtail.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_headtail.sh b/tests/test_headtail.sh index 2ebb80933..8f7ceb05b 100755 --- a/tests/test_headtail.sh +++ b/tests/test_headtail.sh @@ -25,6 +25,7 @@ echo " cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh until [ $# -lt 1 ]; do @@ -34,7 +35,7 @@ until [ $# -lt 1 ]; do done data_dir="tests/data" -broken_dir="$data_dir/broken_json_data" +#broken_dir="$data_dir/broken_json_data" testfile="$data_dir/plant_catalog.xml" @@ -42,7 +43,7 @@ check(){ cmd="$1" expected="$2" msg="$3" - output="$(eval $cmd)" + output="$(eval "$cmd")" result="$(cksum <<< "$output")" echo -n "checking headtail $msg => " if [ "$result" = "$expected" ]; then @@ -52,7 +53,7 @@ check(){ echo echo "full output: " echo - eval $cmd + eval "$cmd" echo echo "cksum: $result" exit 1 From 0b95aa636e38e2d66a40417986f5052e91821144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:08:38 +0000 Subject: [PATCH 0221/2295] updated test_json.sh --- tests/test_json.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_json.sh b/tests/test_json.sh index e7aafae63..9bc12780f 100755 --- a/tests/test_json.sh +++ b/tests/test_json.sh @@ -25,6 +25,7 @@ echo " cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh until [ $# -lt 1 ]; do From a452171b06cd5c1b803e39ed20d66596f9c5b145 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:09:19 +0000 Subject: [PATCH 0222/2295] updated test_json.sh --- tests/test_json.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_json.sh b/tests/test_json.sh index 9bc12780f..9e86b0984 100755 --- a/tests/test_json.sh +++ b/tests/test_json.sh @@ -36,9 +36,10 @@ done # ignore multi-line json data file for spark testing for jsonFile in $(find "${1:-.}" -iname '*.json' | - grep -v '/spark-.*-bin-hadoop.*/' | - grep -v 'multirecord.json' | - grep -v -e 'broken' -e 'error'); do + grep -v -e '/spark-.*-bin-hadoop.*/' \ + -e 'multirecord.json' \ + -e 'broken' \ + -e 'error'); do echo "testing json file: $jsonFile" python -mjson.tool < "$jsonFile" > /dev/null done From 8ddc03a543abc1aa9e7447d33140b7e60dc0b45f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:10:24 +0000 Subject: [PATCH 0223/2295] updated test_json_docs_to_bulk_multiline.sh --- tests/test_json_docs_to_bulk_multiline.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_json_docs_to_bulk_multiline.sh b/tests/test_json_docs_to_bulk_multiline.sh index 333688b75..2e81b392a 100755 --- a/tests/test_json_docs_to_bulk_multiline.sh +++ b/tests/test_json_docs_to_bulk_multiline.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing json_docs_to_bulk_multiline.py" @@ -85,6 +86,7 @@ echo "testing stdin" ./json_docs_to_bulk_multiline.py - < "$data_dir/test.json" > "$stdout" ./json_docs_to_bulk_multiline.py < "$data_dir/test.json" > "$stdout" echo "testing stdin and file mix" +# shellcheck disable=SC2094 ./json_docs_to_bulk_multiline.py "$data_dir/test.json" - < "$data_dir/test.json" > "$stdout" # ================================================== @@ -102,10 +104,11 @@ check_broken(){ filename="$1" expected_exitcode="${2:-2}" set +e - ./json_docs_to_bulk_multiline.py "$filename" ${@:3} 2> "$stderr" > "$stdout" + # shellcheck disable=SC2086 + ./json_docs_to_bulk_multiline.py "$filename" ${*:3} 2> "$stderr" > "$stdout" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ $exitcode = "$expected_exitcode" ]; then echo "successfully detected broken json in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From be26a228d4b37bb46e77e22aac951e0ba505b7d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:10:44 +0000 Subject: [PATCH 0224/2295] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 320cb86e8..0b20f8609 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "JSON => XML" From c17419487c5665e682e96b83ea3d0887919a9ce2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:12:00 +0000 Subject: [PATCH 0225/2295] updated test_git_check_branches_upstream.sh --- tests/test_git_check_branches_upstream.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh index 878529a01..35a818eb8 100755 --- a/tests/test_git_check_branches_upstream.sh +++ b/tests/test_git_check_branches_upstream.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Git check branches upstream" From e0a99def1b2bd6270ab5c970206a166ae311cf6a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:14:05 +0000 Subject: [PATCH 0226/2295] updated test_opentsdb.sh --- tests/test_opentsdb.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_opentsdb.sh b/tests/test_opentsdb.sh index 70ceb7e3d..ac98e609a 100755 --- a/tests/test_opentsdb.sh +++ b/tests/test_opentsdb.sh @@ -16,14 +16,12 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$srcdir/.." -cd "$srcdir2/.." - -. "$srcdir2/utils.sh" - -srcdir="$srcdir2" +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" echo " # ============================================================================ # @@ -41,7 +39,7 @@ export ZOOKEEPER_PORT=2181 export OPENTSDB_PORTS="$ZOOKEEPER_PORT $HBASE_STARGATE_PORT 8085 $HBASE_THRIFT_PORT 9095 3000 16000 16010 16201 16301" export OPENTSDB_TEST_PORTS="$ZOOKEEPER_PORT $HBASE_THRIFT_PORT 3000" -export OPENTSDB_VERSIONS="${@:-latest}" +export OPENTSDB_VERSIONS="${*:-latest}" #export DOCKER_IMAGE="opower/opentsdb" #export DOCKER_IMAGE="petergrace/opentsdb-docker" @@ -68,11 +66,13 @@ generate_test_data(){ #chars="$(echo {A..Z} {a..z} {0..9})" chars=$(echo {A..Z} | tr -d ' ') ts="$(date '+%s')" + # shellcheck disable=SC2034 for x in {1..100}; do + # shellcheck disable=SC2034 for y in {1..1000}; do metric="metric${chars:$((RANDOM % ${#chars})):1}" for z in {1..5}; do - echo "ship${RANDOM:0:3} $(($ts + $RANDOM)) $RANDOM id=$metric crew=$z" + echo "ship${RANDOM:0:3} $((ts + RANDOM)) $RANDOM id=$metric crew=$z" done done done > "$DATA_FILE" @@ -111,6 +111,7 @@ made up error line EOF hr echo "testing from data file and STDIN at the same time:" + # shellcheck disable=SC2094 ./opentsdb_import_metric_distribution.py --key-prefix-length 7 "$DATA_FILE" - < "$DATA_FILE" hr @@ -119,7 +120,7 @@ EOF } for version in $OPENTSDB_VERSIONS; do - test_opentsdb $version + test_opentsdb "$version" done if [ -z "${NODELETE:-}" ]; then echo -n "removing test data: " From 8538d8bfb96b576c719d273ad69e0f354392b7d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:15:46 +0000 Subject: [PATCH 0227/2295] updated test_presto.sh --- tests/test_presto.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_presto.sh b/tests/test_presto.sh index 7b47934f5..ba9add032 100755 --- a/tests/test_presto.sh +++ b/tests/test_presto.sh @@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "P r e s t o S Q L" export PRESTO_TERADATA_VERSIONS="latest 0.152 0.157 0.167 0.179" -export PRESTO_VERSIONS="${@:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}" +export PRESTO_VERSIONS="${*:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}" PRESTO_HOST="${DOCKER_HOST:-${PRESTO_HOST:-${HOST:-localhost}}}" PRESTO_HOST="${PRESTO_HOST##*/}" @@ -64,8 +65,8 @@ test_presto2(){ when_url_content "http://$PRESTO_HOST:$PRESTO_PORT/v1/service/presto/general" nodeId hr expected_version="$version" - if [ "$version" = "latest" -o \ - "$version" = "NODOCKER" ]; then + if [ "$version" = "latest" ] || + [ "$version" = "NODOCKER" ]; then if [ "$teradata_distribution" = 1 ]; then echo "latest version, fetching latest version from DockerHub master branch" expected_version="$(dockerhub_latest_version presto)" @@ -82,6 +83,7 @@ test_presto2(){ hr PRESTO_PORT="$PRESTO_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_presto_coordinator.py $non_presto_node1 $non_presto_node2 + # shellcheck disable=SC2097,SC2098 PRESTO_PORT="$PRESTO_PORT_DEFAULT" run_grep "^$PRESTO_HOST:$PRESTO_PORT$" ./find_active_presto_coordinator.py $non_presto_node1 "$PRESTO_HOST:$PRESTO_PORT" echo "Completed $run_count Presto tests" @@ -117,11 +119,12 @@ test_presto(){ fi done fi - if [ "$teradata_distribution" = "1" -a $facebook_only -eq 0 ]; then + if [ "$teradata_distribution" = "1" ] && + [ $facebook_only -eq 0 ]; then echo "Testing Teradata's Presto distribution version: '$version'" COMPOSE_FILE="$srcdir/docker/presto-docker-compose.yml" test_presto2 "$version" # must call this manually here as we're sneaking in an extra batch of tests that run_test_versions is generally not aware of - let total_run_count+=$run_count + ((total_run_count+=run_count)) # reset this so it can be used in test_presto to detect now testing Facebook teradata_distribution=0 fi From 80c0d309d8cfc3a238e4021bb7c1e32fb6df7477 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:16:43 +0000 Subject: [PATCH 0228/2295] updated test_quay_show_tags.sh --- tests/test_quay_show_tags.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh index c9c55ad62..6765bc9e9 100755 --- a/tests/test_quay_show_tags.sh +++ b/tests/test_quay_show_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Quay.io Show Tags" @@ -38,6 +40,8 @@ echo echo echo "All Quay Show Tags tests completed successfully" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "Quay Show Tags tests completed in" echo From 3f453cf509c37fd6d81feb982801511440f0cb6a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:17:44 +0000 Subject: [PATCH 0229/2295] updated test_solrcloud.sh --- tests/test_solrcloud.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/test_solrcloud.sh b/tests/test_solrcloud.sh index bafb97a47..31df34f43 100755 --- a/tests/test_solrcloud.sh +++ b/tests/test_solrcloud.sh @@ -15,17 +15,16 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$srcdir2/.." +cd "$srcdir/.." +# shellcheck disable=SC1091 . ./tests/utils.sh -srcdir="$srcdir2" - section "S o l r C l o u d" -export SOLRCLOUD_VERSIONS="${@:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}" +export SOLRCLOUD_VERSIONS="${*:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}" SOLR_HOST="${DOCKER_HOST:-${SOLR_HOST:-${HOST:-localhost}}}" SOLR_HOST="${SOLR_HOST##*/}" @@ -46,11 +45,9 @@ trap_debug_env solr zookeeper test_solrcloud(){ local version="$1" # SolrCloud 4.x needs some different args / locations - if [ ${version:0:1} = 4 ]; then - four=true + if [ "${version:0:1}" = 4 ]; then export SOLR_COLLECTION="collection1" else - four="" export SOLR_COLLECTION="gettingstarted" fi section2 "Setting up SolrCloud $version docker test container" @@ -63,7 +60,8 @@ test_solrcloud(){ hr when_url_content "http://$SOLR_HOST:$SOLR_PORT/solr/" "Solr Admin" hr - local DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')" + local DOCKER_CONTAINER + DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')" echo "container is $DOCKER_CONTAINER" if [ -n "${NOTESTS:-}" ]; then exit 0 @@ -73,6 +71,7 @@ test_solrcloud(){ hr SOLR_PORT="$SOLR_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2 + # shellcheck disable=SC2097,SC2098 SOLR_PORT="$SOLR_PORT_DEFAULT" run_grep "^$SOLR_HOST:$SOLR_PORT$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2 "$SOLR_HOST:$SOLR_PORT" docker-compose down From 8473a3acbbf825dfca772330683d971adc4a2de8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:18:14 +0000 Subject: [PATCH 0230/2295] updated test_serf_event_handler.sh --- tests/test_serf_event_handler.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh index 00acd0fe7..eee228571 100755 --- a/tests/test_serf_event_handler.sh +++ b/tests/test_serf_event_handler.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Serf Event Handler" From 3650ae179993319b7266164cd3cd73684d5064b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:19:48 +0000 Subject: [PATCH 0231/2295] updated test_spark_csv_to_avro.sh --- tests/test_spark_csv_to_avro.sh | 41 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh index 05cf1bc17..dc43c31d8 100755 --- a/tests/test_spark_csv_to_avro.sh +++ b/tests/test_spark_csv_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark CSV => Avro" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" # don't support Spark <= 1.3 due to difference in databricks avro dependency for SPARK_VERSION in $SPARK_VERSIONS; do @@ -57,24 +58,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-header-$dir.avro" - ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-header-schemaoverride-$dir.avro" - ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float && - echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" || - { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float; then + echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" + else + echo "FAILED with header and schema override with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-$dir.avro" - ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro" && - echo "SUCCEEDED with no header with Spark $SPARK_VERSION" || - { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro"; then + echo "SUCCEEDED with no header with Spark $SPARK_VERSION" + else + echo "FAILED with no header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-types-$dir.avro" - ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro" && - echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" || - { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro"; then + echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" + else + echo "FAILED with no header and float type with Spark $SPARK_VERSION" + exit 1 + fi # if [ "$(cksum < "test-header-$dir.avro/part-r-00001.avro")" = "$(cksum < "test-noheader-$dir.avro/part-r-00001.avro")" ]; then # echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv avro output" From c637d656112137c899cb57f66a1d1eca4b96e912 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:21:11 +0000 Subject: [PATCH 0232/2295] updated test_spark_csv_to_parquet.sh --- tests/test_spark_csv_to_parquet.sh | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh index 1fc4b6b55..7f95341e1 100755 --- a/tests/test_spark_csv_to_parquet.sh +++ b/tests/test_spark_csv_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark CSV => Parquet" @@ -29,7 +30,7 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" +export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -54,24 +55,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-header-$dir.parquet" - ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-header-schemaoverride-$dir.parquet" - ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float && - echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" || - { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float; then + echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" + else + echo "FAILED with header and schema override with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-$dir.parquet" - ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet" && - echo "SUCCEEDED with no header with Spark $SPARK_VERSION" || - { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet"; then + echo "SUCCEEDED with no header with Spark $SPARK_VERSION" + else + echo "FAILED with no header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-types-$dir.parquet" - ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet" && - echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" || - { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet"; then + echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" + else + echo "FAILED with no header and float type with Spark $SPARK_VERSION" + exit 1 + fi # if [ "$(cksum < "test-header-$dir.parquet/part-r-00001.parquet")" = "$(cksum < "test-noheader-$dir.parquet/part-r-00001.parquet")" ]; then # echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv parquet output" From 536e02325ce6a7882aa9fb5d5f0922aabec582bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:21:47 +0000 Subject: [PATCH 0233/2295] updated test_spark_json_to_avro.sh --- tests/test_spark_json_to_avro.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh index 7c511eb3a..d6506a50c 100755 --- a/tests/test_spark_json_to_avro.sh +++ b/tests/test_spark_json_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark JSON => Avro" @@ -30,9 +31,9 @@ if is_inside_docker; then fi # don't support Spark <= 1.3 due to difference in databricks avro dependency -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires upgrade to spark-avro 3.0.0 -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -57,9 +58,12 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-$dir.avro" - ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi #../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" -s Year:String,Make,Model,Dimension.0.Length:float && # echo "SUCCEEDED with header with Spark $SPARK_VERSION" || From c1f982d207c4e46463483eaa60299af14e9f81d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:23:17 +0000 Subject: [PATCH 0234/2295] updated test_spark_json_to_parquet.sh --- tests/test_spark_json_to_parquet.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh index 170631ba8..9a4a8bc37 100755 --- a/tests/test_spark_json_to_parquet.sh +++ b/tests/test_spark_json_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark JSON => Parquet" @@ -29,7 +30,7 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" +export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -50,8 +51,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.parquet" - ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 2fe6cf02d1ab155348702fa6dd5fc479bd66a291 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:23:55 +0000 Subject: [PATCH 0235/2295] updated test_spark_z_avro_to_parquet.sh --- tests/test_spark_z_avro_to_parquet.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh index ed20eb34a..02fc1ea26 100755 --- a/tests/test_spark_z_avro_to_parquet.sh +++ b/tests/test_spark_z_avro_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark Avro => Parquet" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.parquet" - ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 973c91007dd934cf18c272a974c004d7e8ab1b24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:24:24 +0000 Subject: [PATCH 0236/2295] updated test_spark_z_parquet_to_avro.sh --- tests/test_spark_z_parquet_to_avro.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh index be4f764f8..6e88f0cfc 100755 --- a/tests/test_spark_z_parquet_to_avro.sh +++ b/tests/test_spark_z_parquet_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark Parquet => Avro" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.avro" - ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 3a6d704fd8cfe62b13f930972d34eb9a7b2a5114 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 18:21:20 +0000 Subject: [PATCH 0237/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 829cf04b0..55da73ff4 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -514,6 +514,11 @@ dest[138]=' aws elb create-load-balancer --load-balancer-name Date: Wed, 22 Jan 2020 18:27:43 +0000 Subject: [PATCH 0238/2295] updated anonymize.py --- anonymize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/anonymize.py b/anonymize.py index 07bc730ac..e0904b498 100755 --- a/anonymize.py +++ b/anonymize.py @@ -306,6 +306,8 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(?. anyway by later fqdn anonymization + #'aws10': r'ec2-\d+-\d+-\d+-\d+\.{region}(\.compute\.amazonaws\.com)'.format(region='[A-Za-z0-9-]+'), 'db': r'({switch_prefix}(?:db|database)-?name{arg_sep})\S+'\ .format(arg_sep=arg_sep, switch_prefix=switch_prefix), @@ -449,6 +451,7 @@ def __init__(self): 'aws7': r'', 'aws8': r'\1:///', 'aws9': r'', + #'aws10': r'ec2-x-x-x-x.\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From a85da022d15d2b14653607e3c9e9a8743f221453 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:13:06 +0000 Subject: [PATCH 0239/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 55da73ff4..cb781e11e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -519,6 +519,9 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88 #dest[140]='ssh -i myKey -N -L 8888::8888 @' dest[140]='ssh -i myKey -N -L 8888::8888 @' +src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 7a857e497fafb1f43afa09b1532232d1a2754561 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:13:08 +0000 Subject: [PATCH 0240/2295] added hive warehouse db/table match and tightened domain\user format to avoid newlines and carriage returns --- anonymize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index e0904b498..106f73952 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.8' +__version__ = '0.10.9' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -320,6 +320,7 @@ def __init__(self): id_or_name=id_or_name, switch_prefix=switch_prefix), 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', + 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, @@ -357,7 +358,7 @@ def __init__(self): 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), - 'user4': r'(?/) exclude patterns '>/' where we have already matched and token replaced @@ -456,6 +457,7 @@ def __init__(self): 'db2': r'\1', 'db3': r'\1', 'db4': r'\1<\2>', + 'db5': r'/user/hive/warehouse/.db/
', 'generic': r'\1://', 'generic2': r'\1', 'generic3': r'\1', From 6f2269dd23ea84ca433a495b36283d70e07019c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:17:36 +0000 Subject: [PATCH 0241/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index cb781e11e..919d91e95 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -519,8 +519,8 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88 #dest[140]='ssh -i myKey -N -L 8888::8888 @' dest[140]='ssh -i myKey -N -L 8888::8888 @' -src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" -dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" # TODO: move proxy hosts to host matches and re-enable From 65237f5e3d264f61026ecb8666d6a8e026b508cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:21:52 +0000 Subject: [PATCH 0242/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 919d91e95..5f7657ea8 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -399,9 +399,9 @@ src[102]="-Dhost.domain.com=blah" dest[102]="-Dhost.domain.com=blah" # check escape codes get stripped if present (eg. if piping from grep --color-yes) -#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" -#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443" -src[103]="$(echo somehost:443 | grep --color=yes host)" +# breaks test_anonymize.py which doesn't eval this, so put it explicitly +#src[103]="$(echo somehost:443 | grep --color=yes host)" +src[103]="somehost:443" dest[103]=":443" src[104]='..., "user": "blah", "group": "blah2", "host": "blah3", ...' From 082120243e8e4d65073748cd4b17f0b9e967b839 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:28:45 +0000 Subject: [PATCH 0243/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 116 ++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 5f7657ea8..72ded4847 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -40,40 +40,6 @@ start_time="$(start_timer "$anonymize")" # Custom Tests # ============================================================================ # -if [ -z "$test_nums" ]; then - echo - echo "Running Custom Tests:" - echo - echo "checking file args:" - run++ - if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then - echo "SUCCEEDED - anonymized README.md > 100 lines" - else - echo "FAILED - suspicious README.md file arg result came to <= 100 lines" - exit 1 - fi - hr - - run_grep "@" $anonymize --email <<< "hari@domain.com" - run_grep "@" $anonymize -E <<< "hari@domain.com" - - run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" - run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" - run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" - run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" - run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" - run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" - run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" - run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" - run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" - - run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" - run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' - run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah' -fi - -# ============================================================================ # - src[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password=somep@ssword! -h myHost.internal -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on 'host.domain.com' (111)" dest[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password= -h -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on '' (111)" @@ -578,22 +544,20 @@ run_tests(){ for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } [ -n "${dest[$i]:-}" ] || { echo "code error: dest[$i] not defined"; exit 1; } - if [ -n "$parallel" ]; then - test_anonymize "${src[$i]}" "${dest[$i]}" & - else - test_anonymize "${src[$i]}" "${dest[$i]}" - fi + #test_anonymize "${src[$i]}" "${dest[$i]}" + run++ done + "$srcdir/test_anonymize.py" } -echo -echo "Running Standard Tests with --all --skip-exceptions" -echo +#echo +#echo "Running Standard Tests with --all --skip-exceptions" +#echo run_tests # ignore_run_unqualified -echo -echo "Running Tests preseving text without --network enabled:" -echo +#echo +#echo "Running Tests preseving text without --network enabled:" +#echo # check normal don't strip these src[901]="reading password from foo" dest[901]="reading password from foo" @@ -602,11 +566,11 @@ src[902]="some description = blah, module = foo" dest[902]="some description = blah, module = foo" args="-HKEiu" -run_tests 901 902 # ignore_run_unqualified +#run_tests 901 902 # ignore_run_unqualified -echo -echo "Running Network Specific Tests:" -echo +#echo +#echo "Running Network Specific Tests:" +#echo # now check --network / --cisco / --juniper do strip these src[903]="reading password from bar" dest[903]="reading password " @@ -615,16 +579,50 @@ src[904]="some description = blah, module=bar" dest[904]="some description " args="--network" -run_tests 903 904 # ignore_run_unqualified - -if [ -n "$parallel" ]; then - # can't trust exit code for parallel yet, only for quick local testing - exit 1 -# for i in ${!src[@]}; do -# let j=$i+1 -# wait %$j -# [ $? -eq 0 ] || { echo "FAILED"; exit $?; } -# done +#run_tests 903 904 # ignore_run_unqualified + +#if [ -n "$parallel" ]; then +# # can't trust exit code for parallel yet, only for quick local testing +# exit 1 +## for i in ${!src[@]}; do +## let j=$i+1 +## wait %$j +## [ $? -eq 0 ] || { echo "FAILED"; exit $?; } +## done +#fi + +# ============================================================================ # + +if [ -z "$test_nums" ]; then + echo + echo "Running Custom Tests:" + echo + echo "checking file args:" + run++ + if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then + echo "SUCCEEDED - anonymized README.md > 100 lines" + else + echo "FAILED - suspicious README.md file arg result came to <= 100 lines" + exit 1 + fi + hr + + run_grep "@" $anonymize --email <<< "hari@domain.com" + run_grep "@" $anonymize -E <<< "hari@domain.com" + + run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" + run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" + run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" + run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" + run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" + run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" + run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" + run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" + run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" + + run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" + run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' + run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah' fi echo From f9ec043ff08fba90769a492bf37298b4ea83b276 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:29:28 +0000 Subject: [PATCH 0244/2295] added tests/test_anonymize.py --- tests/test_anonymize.py | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 tests/test_anonymize.py diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py new file mode 100755 index 000000000..8cddda652 --- /dev/null +++ b/tests/test_anonymize.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import re +#import StringIO +import subprocess +from subprocess import PIPE +import sys + +srcdir = os.path.abspath(os.path.dirname(__file__)) + +anonymize_test_sh = os.path.join(srcdir, 'test_anonymize.sh') + +anonymize = '{}/../anonymize.py'.format(srcdir) + +src = {} +dest = {} + +src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$') +dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$') +args_regex = re.compile(r'^args=["\'](.+)["\']\s*$') + +def normalize_text(text): + text = text.replace(r'\"', '"') + text = text.replace(r"\'", "'") + return text + +def run(): + #test_input = StringIO.StringIO() + #test_input.write('\n'.join(src)) + global src + global dest + src = {int(k) : v for k, v in src.items()} + dest = {int(k) : v for k, v in dest.items()} + src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name + test_input = '\n'.join([src[key] for key in src_keys]) + + print('running anonymize tests using: {} {}'.format(anonymize, args)) + process = subprocess.Popen([anonymize, args], stdin=PIPE, stdout=PIPE) + (stdout, _) = process.communicate(input=test_input) + index = 0 + for line in stdout.split('\n'): # pylint: disable=redefined-outer-name + key = src_keys[index] + _input = src[key] + expected = dest[key] + if line != expected: + print('FAILED to anonymize line during test {}'.format(key)) + print('input: {}'.format(_input)) + print('expected: {}'.format(expected)) + print('got: {}'.format(line)) + sys.exit(1) + print('SUCCEEDED anonymization test {}'.format(key)) + index += 1 + +with open(anonymize_test_sh) as filehandle: + for line in filehandle: + src_match = src_regex.match(line) + if src_match: + key = src_match.group(1) + if key in src: + raise AssertionError('Duplicate key index src[{}]'.format(key)) + value = src_match.group(2) + value = normalize_text(value) + src[key] = value + dest_match = dest_regex.match(line) + if dest_match: + key = dest_match.group(1) + if key in dest: + raise AssertionError('Duplicate key index dest[{}]'.format(key)) + value = dest_match.group(2) + value = normalize_text(value) + dest[key] = value + args_match = args_regex.match(line) + if args_match: + args = args_match.group(1) + run() + src = {} + dest = {} From b2259b32e0ebc9166aa55c8af9b1b8cf2581abb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:30:02 +0000 Subject: [PATCH 0245/2295] updated test_anonymize.py --- tests/test_anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 8cddda652..c7a39e8c0 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -30,8 +30,8 @@ def normalize_text(text): def run(): #test_input = StringIO.StringIO() #test_input.write('\n'.join(src)) - global src - global dest + global src # pylint: disable=global-statement + global dest # pylint: disable=global-statement src = {int(k) : v for k, v in src.items()} dest = {int(k) : v for k, v in dest.items()} src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name From 8a68421602fa8cd42fc8ef35e7abb9b78bac1eb5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:09:17 +0000 Subject: [PATCH 0246/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 72ded4847..d58f7315e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -19,11 +19,11 @@ set -euo pipefail srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" test_nums="${*:-}" -parallel="" -if [ "$test_nums" = "p" ]; then - parallel="1" - test_nums="" -fi +#parallel="" +#if [ "$test_nums" = "p" ]; then +# parallel="1" +# test_nums="" +#fi cd "$srcdir/.."; @@ -553,7 +553,7 @@ run_tests(){ #echo #echo "Running Standard Tests with --all --skip-exceptions" #echo -run_tests # ignore_run_unqualified +run_tests "$@" # ignore_run_unqualified #echo #echo "Running Tests preseving text without --network enabled:" From 0fded27273ba1071f663e17b8d9625c6db6feb33 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:21:41 +0000 Subject: [PATCH 0247/2295] updated test_anonymize.py --- tests/test_anonymize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index c7a39e8c0..8d67fcf36 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -20,7 +20,7 @@ src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$') dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$') -args_regex = re.compile(r'^args=["\'](.+)["\']\s*$') +args_regex = re.compile(r'^\s*args=["\'](.+)["\']\s*$') def normalize_text(text): text = text.replace(r'\"', '"') @@ -38,7 +38,8 @@ def run(): test_input = '\n'.join([src[key] for key in src_keys]) print('running anonymize tests using: {} {}'.format(anonymize, args)) - process = subprocess.Popen([anonymize, args], stdin=PIPE, stdout=PIPE) + cmd = [anonymize] + [_ for _ in args.split()] + process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) (stdout, _) = process.communicate(input=test_input) index = 0 for line in stdout.split('\n'): # pylint: disable=redefined-outer-name From 5991a1432fb3b56f3bc0b64a3626714e6787f95a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:24:52 +0000 Subject: [PATCH 0248/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index d58f7315e..7335c0168 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -610,15 +610,33 @@ if [ -z "$test_nums" ]; then run_grep "@" $anonymize --email <<< "hari@domain.com" run_grep "@" $anonymize -E <<< "hari@domain.com" - run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" - run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" - run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" - run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" - run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" - run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" - run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" - run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" - run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" + src[800]="4.3.2.1" + dest[800]=".1" + + src[801]="4.3.2.1/24" + dest[801]=".1/" + + src[802]="4.3.2.1" + dest[802]=".1" + + src[803]="ip-1-2-3-4" + dest[803]=".4" + + src[804]="ip-1-2-3-4-5" + dest[804]="ip-1-2-3-4-5" + + src[805]="dip-1-2-3-4" + dest[805]="dip-1-2-3-4" + + src[806]="5.4.3.2.1" + dest[806]="5.4.3.2.1" + + src[807]="log4j-1.2.3.4.jar" + dest[807]="log4j-1.2.3.4.jar" + + src[808]="/usr/hdp/2.6.2.0-123" + dest[808]="/usr/hdp/2.6.2.0-123" + args="-a --ip-prefix" run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' From 364cf4e9fe09730ccd7d13fef16c7d394340cc2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:13:00 +0000 Subject: [PATCH 0249/2295] integrated with CLI for better option handling and logging --- hive_schemas_csv.py | 277 ++++++++++++++++++++++---------------------- 1 file changed, 137 insertions(+), 140 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index f93bb9137..e08b5414d 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -56,154 +56,151 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from __future__ import unicode_literals +#from __future__ import unicode_literals -import argparse import csv -import logging import os import socket import sys from impala.dbapi import connect +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser( - description="Dumps all {} schemas, tables, columns and types to CSV format on stdout".format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - # must set type to str otherwise csv module gives this error on Python 2.7: - # TypeError: "delimiter" must be string, not unicode - parser.add_argument('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)') - parser.add_argument('-Q', '--quotechar', default='"', type=str, - help='Generate quoted CSV (recommended, default is double quote \'"\')') - parser.add_argument('-E', '--escapechar', help='Escape char if needed') - parser.add_argument('-v', '--verbose', action='count', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - log.debug('kerberos enabled') - log.debug('krb5 remote service principal name = %s', args.krb5_service_name) - if args.ssl is True: - log.debug('ssl enabled') - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - conn = connect_db(args, 'default') - - quoting = csv.QUOTE_ALL - if args.quotechar == '': - quoting = csv.QUOTE_NONE - fieldnames = ['database', 'table', 'column', 'type'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=args.delimiter, - quotechar=args.quotechar, - escapechar=args.escapechar, - quoting=quoting, - fieldnames=fieldnames) - csv_writer.writeheader() - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - for table_row in table_cursor: - table = table_row[0] - log.info('describing table %s', table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) - for column_row in column_cursor: - column = column_row[0] - column_type = column_row[1] - csv_writer.writerow({'database': database, - 'table': table, - 'column': column, - 'type': column_type}) +__version__ = '0.4.0' + + +class HiveSchemasCSV(CLI): + + def __init__(self): + # Python 2.x + super(HiveSchemasCSV, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['HiveServer2', 'Hive'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 10000 + self.default_service_name = 'hive' + self.kerberos = False + self.krb5_service_name = self.default_service_name + self.ssl = False + self.delimiter = None + self.quotechar = None + self.escapechar = None + + if 'impala' in sys.argv[0]: + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + self.env_prefixes = ['IMPALA'] + + def add_options(self): + super(HiveSchemasCSV, self).add_options() + self.add_hostoption() + self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, + help='Service principal (default: {})'.format(self.default_service_name)) + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' + # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour + self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)') + self.add_opt('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV (recommended, default is double quote \'"\')') + self.add_opt('-E', '--escapechar', help='Escape char if needed') + + def process_options(self): + super(HiveSchemasCSV, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.kerberos = self.get_opt('kerberos') + self.krb5_service_name = self.get_opt('krb5_service_name') + self.ssl = self.get_opt('ssl') + self.delimiter = self.get_opt('delimiter') + self.quotechar = self.get_opt('quotechar') + self.escapechar = self.get_opt('escapechar') + + def connect(self, database): + auth_mechanism = None + if self.kerberos: + auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', self.krb5_service_name) + if self.ssl: + log.debug('ssl enabled') + + log.info('connecting to %s:%s database %s', self.host, self.port, database) + return connect( + host=self.host, + port=self.port, + auth_mechanism=auth_mechanism, + use_ssl=self.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=self.krb5_service_name + ) + + def run(self): + + conn = self.connect('default') + + quoting = csv.QUOTE_ALL + if self.quotechar == '': + quoting = csv.QUOTE_NONE + fieldnames = ['database', 'table', 'column', 'type'] + csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + csv_writer.writeheader() + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + log.info('describing table %s', table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use {}'.format(database)) + column_cursor.execute('describe {}'.format(table)) + for column_row in column_cursor: + column = column_row[0] + column_type = column_row[1] + csv_writer.writerow({'database': database, + 'table': table, + 'column': column, + 'type': column_type}) if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveSchemasCSV().main() From 031ea7c943c493ecb80f5a2b825d036799a1978e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:18:12 +0000 Subject: [PATCH 0250/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index e08b5414d..44d64c981 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -99,12 +99,6 @@ def __init__(self): self.quotechar = None self.escapechar = None - if 'impala' in sys.argv[0]: - self.name = 'Impala' - self.default_port = 21050 - self.default_service_name = 'impala' - self.env_prefixes = ['IMPALA'] - def add_options(self): super(HiveSchemasCSV, self).add_options() self.add_hostoption() From 737b4a1849ab169450850015e745dd538c028339 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:18:33 +0000 Subject: [PATCH 0251/2295] subclassed impala_schemas_csv.py --- impala_schemas_csv.py | 81 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) mode change 120000 => 100755 impala_schemas_csv.py diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py deleted file mode 120000 index baed3c963..000000000 --- a/impala_schemas_csv.py +++ /dev/null @@ -1 +0,0 @@ -hive_schemas_csv.py \ No newline at end of file diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py new file mode 100755 index 000000000..e6ac0ef90 --- /dev/null +++ b/impala_schemas_csv.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout + +In practice Hive is much more reliable for dumping masses of schema + +Impala appears faster initially but then slows down more than Hive and hits things query handle errors +under sustained load of extracting large amounts of schema information + +There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this + + +CSV format: + +database,table,column,type + + +I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2) +which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but +if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will +raise a traceback to tell you to set one (eg. --escapechar='\\') + +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +from hive_schemas_csv import HiveSchemasCSV + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaSchemasCSV(HiveSchemasCSV): + + def __init__(self): + # Python 2.x + super(ImpalaSchemasCSV, self).__init__() + # Python 3.x + # super().__init__() + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaSchemasCSV().main() From 70c9929895084a2cf835bb66b386e8b8d11f35d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:45:14 +0000 Subject: [PATCH 0252/2295] updated impala_schemas_csv.py --- impala_schemas_csv.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index e6ac0ef90..2b6b19fdc 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -18,7 +18,7 @@ Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -61,7 +61,7 @@ from hive_schemas_csv import HiveSchemasCSV __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' class ImpalaSchemasCSV(HiveSchemasCSV): @@ -71,9 +71,11 @@ def __init__(self): super(ImpalaSchemasCSV, self).__init__() # Python 3.x # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class self.name = 'Impala' - self.default_port = 21050 - self.default_service_name = 'impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' if __name__ == '__main__': From 93d95469797dc051d4ae8813f3c5577119828698 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:45:39 +0000 Subject: [PATCH 0253/2295] refactored out to HiveImpalaCLI --- hive_schemas_csv.py | 64 ++++++++------------------------------------- 1 file changed, 11 insertions(+), 53 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 44d64c981..5b8d6c3eb 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -16,9 +16,9 @@ """ -Connect to a HiveServer2 or Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout +Connect to HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -60,15 +60,16 @@ import csv import os -import socket import sys -from impala.dbapi import connect -libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) -sys.path.append(libdir) +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_host, validate_port - from harisekhon import CLI + from harisekhon.utils import log + from hive_impala_cli import HiveImpalaCLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -76,36 +77,22 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' -class HiveSchemasCSV(CLI): +class HiveSchemasCSV(HiveImpalaCLI): def __init__(self): # Python 2.x super(HiveSchemasCSV, self).__init__() # Python 3.x # super().__init__() - self.name = ['HiveServer2', 'Hive'] - self.host = None - self.port = None - self.default_host = socket.getfqdn() - self.default_port = 10000 - self.default_service_name = 'hive' - self.kerberos = False - self.krb5_service_name = self.default_service_name - self.ssl = False self.delimiter = None self.quotechar = None self.escapechar = None def add_options(self): super(HiveSchemasCSV, self).add_options() - self.add_hostoption() - self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, - help='Service principal (default: {})'.format(self.default_service_name)) - self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: # TypeError: "delimiter" must be string, not unicode # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' @@ -117,39 +104,10 @@ def add_options(self): def process_options(self): super(HiveSchemasCSV, self).process_options() - self.host = self.get_opt('host') - self.port = self.get_opt('port') - validate_host(self.host) - validate_port(self.port) - self.port = int(self.port) - self.kerberos = self.get_opt('kerberos') - self.krb5_service_name = self.get_opt('krb5_service_name') - self.ssl = self.get_opt('ssl') self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') - def connect(self, database): - auth_mechanism = None - if self.kerberos: - auth_mechanism = 'GSSAPI' - log.debug('kerberos enabled') - log.debug('krb5 remote service principal name = %s', self.krb5_service_name) - if self.ssl: - log.debug('ssl enabled') - - log.info('connecting to %s:%s database %s', self.host, self.port, database) - return connect( - host=self.host, - port=self.port, - auth_mechanism=auth_mechanism, - use_ssl=self.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=self.krb5_service_name - ) - def run(self): conn = self.connect('default') From 75905a0b7937262dce1820190ddf8bb7495c01c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:49:54 +0000 Subject: [PATCH 0254/2295] updated .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f0ca6569a..3722dc711 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ # breaks local lib/ change tracking lib64/ parts/ sdist/ From b2de84ef84db86f4cd5d54a273d7c7062782a60a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:50:01 +0000 Subject: [PATCH 0255/2295] added lib/hive_impala_cli.py --- lib/hive_impala_cli.py | 103 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 lib/hive_impala_cli.py diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py new file mode 100755 index 000000000..d5f9a2a06 --- /dev/null +++ b/lib/hive_impala_cli.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import socket +import sys +from impala.dbapi import connect +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveImpalaCLI(CLI): + + def __init__(self): + # Python 2.x + super(HiveImpalaCLI, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['HiveServer2', 'Hive'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 10000 + self.default_service_name = 'hive' + self.kerberos = False + self.krb5_service_name = self.default_service_name + self.ssl = False + #self.timeout_default = 86400 + self.timeout_default = None + if 'impala' in sys.argv[0]: + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + + def add_options(self): + super(HiveImpalaCLI, self).add_options() + self.add_hostoption() + self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, + help='Service principal (default: {})'.format(self.default_service_name)) + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') + + def process_options(self): + super(HiveImpalaCLI, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.kerberos = self.get_opt('kerberos') + self.krb5_service_name = self.get_opt('krb5_service_name') + self.ssl = self.get_opt('ssl') + + def connect(self, database): + auth_mechanism = None + if self.kerberos: + auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', self.krb5_service_name) + if self.ssl: + log.debug('ssl enabled') + + log.info('connecting to %s:%s database %s', self.host, self.port, database) + return connect( + host=self.host, + port=self.port, + auth_mechanism=auth_mechanism, + use_ssl=self.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=self.krb5_service_name + ) From 64cf017b4329bd99e282bb548426d99c5ee23ad8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:52:33 +0000 Subject: [PATCH 0256/2295] updated impala_schemas_csv.py --- impala_schemas_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 2b6b19fdc..17ce8987b 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -18,7 +18,7 @@ Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable than Impala for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema (see adjacent hive_schemas_csv.py) Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f2878bdbf4f1fd47c6ca4b9e2634a8d3f51cd3e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:52:49 +0000 Subject: [PATCH 0257/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 5b8d6c3eb..efdf62143 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 216e44ab452e86b76f5845ad408c4f281f85eac5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:12:57 +0000 Subject: [PATCH 0258/2295] refactored to use HiveImpalaCLI --- hive_tables_row_counts.py | 314 ++++++++++++++++---------------------- 1 file changed, 132 insertions(+), 182 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index d4d685f26..08eefbfff 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -16,10 +16,10 @@ """ -Connect to a HiveServer2 / Impala node and get rows counts for all tables in all databases, +Connect to HiveServer2 and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -41,71 +41,46 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse -import logging import os import re -import socket import sys import impala -from impala.dbapi import connect +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_impala_cli import HiveImpalaCLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser( - description="Gets row counts for all {} tables / partitions matching database / table / partition regexes"\ - .format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +__version__ = '0.5.0' + + +class HiveTablesRowCounts(HiveImpalaCLI): + + def __init__(self): + # Python 2.x + super(HiveTablesRowCounts, self).__init__() + # Python 3.x + # super().__init__() + self.database = None + self.table = None + self.partition = None + self.ignore_errors = False + + def add_options(self): + super(HiveTablesRowCounts, self).add_options() + self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') + self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') + self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') # # ignore tables that fail with errors like: # @@ -118,130 +93,105 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue + self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') + + def process_options(self): + super(HiveTablesRowCounts, self).process_options() + self.database = self.get_opt('database') + self.table = self.get_opt('table') + self.partition = self.get_opt('partition') + self.ignore_errors = self.get_opt('ignore_errors') + validate_regex(self.database, 'database') + validate_regex(self.table, 'table') + validate_regex(self.partition, 'partition') + + def run(self): + database_regex = re.compile(self.database, re.I) + table_regex = re.compile(self.table, re.I) + partition_regex = re.compile(self.partition, re.I) + conn = self.connect('default') + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, self.database) + continue + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: try: - get_row_counts(conn, args, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if args.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): continue raise - -def get_row_counts(conn, args, database, table, partition_regex): - log.info("getting partitions for database '%s' table '%s'", database, table) - with conn.cursor() as partition_cursor: - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) - try: - partition_cursor.execute('show partitions {table}'.format(table=table)) - for partitions_row in partition_cursor: - partition_key = partitions_row[0] - partition_value = partitions_row[1] - if not partition_regex.match(partition_value): - log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + - "value does not match regex '%s'", - database, - table, - partition_key, - partition_value, - args.partition) - continue - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ - .format(db=database, table=table, key=partition_key, value=partition_value)) - for result in partition_cursor: - row_count = result[0] - print('{db}.{table}.{key}={value}\t{row_count}'.format(\ - db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - # Hive impala.error.HiveServer2Error: is not a partitioned table - # Impala impala.error.HiveServer2Error: Table is not partitioned - if 'is not a partitioned table' not in str(_) and \ - 'Table is not partitioned' not in str(_): - raise - log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", - database, table) - with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) - for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + def get_row_counts(self, conn, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + self.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, + table=table, + key=partition_key, + value=partition_value, + row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # Hive impala.error.HiveServer2Error: is not a partitioned table + # Impala impala.error.HiveServer2Error: Table is not partitioned + if 'is not a partitioned table' not in str(_) and \ + 'Table is not partitioned' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveTablesRowCounts().main() From 1bb4945dfdb66d0279f21e4986de85ff21019c16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:13:25 +0000 Subject: [PATCH 0259/2295] changed to subclass HiveTablesRowCounts --- impala_tables_row_counts.py | 1 - 1 file changed, 1 deletion(-) delete mode 120000 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py deleted file mode 120000 index 65e53c4f2..000000000 --- a/impala_tables_row_counts.py +++ /dev/null @@ -1 +0,0 @@ -hive_tables_row_counts.py \ No newline at end of file From d033db228338a6ca6c0763b29edac9a98bb18633 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:16:02 +0000 Subject: [PATCH 0260/2295] subclassed HiveTablesRowCounts --- impala_tables_row_counts.py | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 100755 index 000000000..e17a88e63 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_counts import HiveTablesRowCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class ImpalaTablesRowCounts(HiveTablesRowCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowCounts().main() From 610d8568a7c1287b64bb98366a05b9ba460660fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:16:27 +0000 Subject: [PATCH 0261/2295] set verbose to only need one -v switch --- lib/hive_impala_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py index d5f9a2a06..7515abddd 100755 --- a/lib/hive_impala_cli.py +++ b/lib/hive_impala_cli.py @@ -55,6 +55,7 @@ def __init__(self): self.kerberos = False self.krb5_service_name = self.default_service_name self.ssl = False + self.verbose_default = 1 #self.timeout_default = 86400 self.timeout_default = None if 'impala' in sys.argv[0]: From 3c8b4984f6172d4e76edf8efb2d0cfff6a0b2d1d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:30:06 +0000 Subject: [PATCH 0262/2295] refactored to subclass HiveImpalaCLI --- hive_foreach_table.py | 276 +++++++++++++++++------------------------- 1 file changed, 113 insertions(+), 163 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 90b294709..79833e7fc 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a HiveServer2 / Impala node and execute a query for all tables in all databases, +Connect to HiveServer2 and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: @@ -26,17 +26,12 @@ hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' -impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' -impala_foreach_table.py --query 'COMPUTE STATS {table}' - or just for today's partition: hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" - -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -58,70 +53,49 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse -import logging import os import re -import socket import sys import impala -from impala.dbapi import connect +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_impala_cli import HiveImpalaCLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching {} table".format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +__version__ = '0.4.0' + + +class HiveForEachTable(HiveImpalaCLI): + + def __init__(self): + # Python 2.x + super(HiveForEachTable, self).__init__() + # Python 3.x + # super().__init__() + self.query = None + self.database = None + self.table = None + self.partition = None + self.ignore_errors = False + + def add_options(self): + super(HiveForEachTable, self).add_options() + self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') + self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') + #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') # # ignore tables that fail with errors like: # @@ -134,107 +108,83 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - query = args.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = args.query.format(table=table) + self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') + + def process_options(self): + super(HiveForEachTable, self).process_options() + self.query = self.get_opt('query') + if not self.query: + self.usage('query not defined') + self.database = self.get_opt('database') + self.table = self.get_opt('table') + #self.partition = self.get_opt('partition') + self.ignore_errors = self.get_opt('ignore_errors') + validate_regex(self.database, 'database') + validate_regex(self.table, 'table') + #validate_regex(self.partition, 'partition') + + def run(self): + database_regex = re.compile(self.database, re.I) + table_regex = re.compile(self.table, re.I) + #partition_regex = re.compile(self.partition, re.I) + conn = self.connect('default') + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, self.database) + continue + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: try: - execute(conn, database, table, query) - except Exception as _: - if args.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): continue raise - -def execute(conn, database, table, query): - try: - log.info(" %s.%s - running %s", database, table, query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - # log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + @staticmethod + def execute(conn, database, table, query): + try: + log.info(" %s.%s - running %s", database, table, query) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) + #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # log.error(_) + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveForEachTable().main() From 2b285e8abf621de8e5f407e632f9eb4f407b78ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:30:16 +0000 Subject: [PATCH 0263/2295] subclassed HiveForEachTable --- impala_foreach_table.py | 92 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) mode change 120000 => 100755 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py deleted file mode 120000 index 44716e6dc..000000000 --- a/impala_foreach_table.py +++ /dev/null @@ -1 +0,0 @@ -hive_foreach_table.py \ No newline at end of file diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 100755 index 000000000..49c8e21e0 --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + +or just for today's partition: + +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" + + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaForEachTable(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(ImpalaForEachTable, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaForEachTable().main() From 15e1475f1c6ead07b2092b39e28e9ca2a233d328 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:54:52 +0000 Subject: [PATCH 0264/2295] added lib/__init__.py --- lib/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/__init__.py diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 000000000..e69de29bb From 79bd12ba42efc5e5202eac0f381bcdc9d22fbf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:57:04 +0000 Subject: [PATCH 0265/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index efdf62143..1dbb3dff3 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -138,7 +138,7 @@ def run(self): table_cursor.execute('show tables') for table_row in table_cursor: table = table_row[0] - log.info('describing table %s', table) + log.info('describing table %s.%s', database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 54f50ff9aa37ac561457edd6f8f7c1332fbd78a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:58:28 +0000 Subject: [PATCH 0266/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 79833e7fc..074a2b55d 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.4.1' class HiveForEachTable(HiveImpalaCLI): @@ -83,6 +83,8 @@ def __init__(self): super(HiveForEachTable, self).__init__() # Python 3.x # super().__init__() + + # self.query can be pre-defined in which case subclassed programs won't expose --query option self.query = None self.database = None self.table = None @@ -91,8 +93,10 @@ def __init__(self): def add_options(self): super(HiveForEachTable, self).add_options() - self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') + # allow subclassing to pre-define query and not expose option in that case + if self.query is None: + self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') @@ -112,7 +116,8 @@ def add_options(self): def process_options(self): super(HiveForEachTable, self).process_options() - self.query = self.get_opt('query') + if self.query is None: + self.query = self.get_opt('query') if not self.query: self.usage('query not defined') self.database = self.get_opt('database') From 399b6bd1933ac2d12bc85b090eef54d381e6a49e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:46:35 +0000 Subject: [PATCH 0267/2295] added hive_tables_null_columns.py --- hive_tables_null_columns.py | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 hive_tables_null_columns.py diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py new file mode 100755 index 000000000..02f47f4ef --- /dev/null +++ b/hive_tables_null_columns.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Describes each table to construct a large query based off all detected columns, +then queries the columns for NULLs and prints the tables columns with all NULLs + +Useful for catching data problems either caused by data input or broken ETL processes + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + +class HiveTablesNullColumns(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesNullColumns, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + sum_part = '' + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use {}'.format(database)) + column_cursor.execute('describe {}'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + sum_part = ', '.join( + ['IF(SUM(IF({col} IS NULL, 1, 0)) = COUNT(*), 1, 0) as {col}'.format(col=column) \ + for column in columns] + ) + query = "SELECT {sum_part} FROM {db}.{table} WHERE "\ + .format(sum_part=sum_part, db=database, table=table) + \ + " IS NULL OR ".join(columns) + " IS NULL" + self.check_table_for_nulls(conn, database, table, columns, query) + + @staticmethod + def check_table_for_nulls(conn, database, table, columns, query): + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + cols_with_nulls = [] + for result in table_cursor: + # tuple of ints (0, 0, 0, .... N) - one per column + for index in range(len(list(result))): + col_result = result[index] + if col_result > 0: + cols_with_nulls.append(columns[index]) + num_cols = len(cols_with_nulls) + total_cols = len(columns) + if cols_with_nulls: + print('WARNING: {db}.{table} has {num}/{total} columns with only NULLs: {cols}'\ + .format(db=database, + table=table, + num=num_cols, + total=total_cols, + cols=', '.join(sorted(cols_with_nulls)) + ) + ) + else: + print('OK: {db}.{table} as {num}/{total} columns with only nulls'\ + .format(db=database, table=table, num=num_cols, total=total_cols)) + + +if __name__ == '__main__': + HiveTablesNullColumns().main() From 11584ab5578e0bc88ecde053d401e336cb0b7f1c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:46:39 +0000 Subject: [PATCH 0268/2295] added impala_tables_null_columns.py --- impala_tables_null_columns.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 impala_tables_null_columns.py diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py new file mode 100755 index 000000000..96cc7a998 --- /dev/null +++ b/impala_tables_null_columns.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Describes each table to construct a large query based off all detected columns, +then queries the columns for NULLs and prints the tables columns with all NULLs + +Useful for catching data problems either caused by data input or broken ETL processes + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_null_columns import HiveTablesNullColumns +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesNullColumns(HiveTablesNullColumns): + + def __init__(self): + # Python 2.x + super(ImpalaTablesNullColumns, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesNullColumns().main() From 42505f841b87f092da43e8812606ea91420be5d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:49:25 +0000 Subject: [PATCH 0269/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c32604247..2b895d3c4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterate all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 9cabf0ce21e91e8e408daa085aa2c93d43b4f9d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:50:14 +0000 Subject: [PATCH 0270/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 02f47f4ef..4da274b88 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -19,10 +19,10 @@ Connect to HiveServer2 and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based off all detected columns, -then queries the columns for NULLs and prints the tables columns with all NULLs +Describes each table to construct a large query based on all detected columns, +then queries the columns for NULLs and prints the tables columns containing only NULLs -Useful for catching data problems either caused by data input or broken ETL processes +Useful for catching problems with data quality or broken ETL processes Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From c99e17fbe443181e6c3c1b2e9034cc4e8d04af52 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:53:44 +0000 Subject: [PATCH 0271/2295] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 96cc7a998..05c149456 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -19,10 +19,10 @@ Connect to an Impala daemon and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based off all detected columns, -then queries the columns for NULLs and prints the tables columns with all NULLs +Describes each table, constructs a complex query to check each column individually for containing only NULLs, +and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching data problems either caused by data input or broken ETL processes +Useful for catching problems with data quality or broken ETL processes Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From 6e1c3d1fbf648515cc2caed71ffb79d19a805f04 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:53:48 +0000 Subject: [PATCH 0272/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 4da274b88..7dab3781e 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -19,8 +19,8 @@ Connect to HiveServer2 and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based on all detected columns, -then queries the columns for NULLs and prints the tables columns containing only NULLs +Describes each table, constructs a complex query to check each column individually for containing only NULLs, +and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns Useful for catching problems with data quality or broken ETL processes From d67733b4cbd17b1a91726c4cad54691010f1ef29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:54:17 +0000 Subject: [PATCH 0273/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b895d3c4..142917fad 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ Environment variables are supported for convenience and also to hide credentials - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterate all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterates all matching tables in all databases (by partition if available) and outputs TSV of table names and row counts + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterates all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From a04139218f34b7ae2f26f7181b9b3163b591ad69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:58:39 +0000 Subject: [PATCH 0274/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index e17a88e63..99f2d5b5a 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -19,6 +19,8 @@ Connect to an Impala daemon and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes +Useful for reconciliations between clusters after migrations + Tested on Impala 2.7.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f3b6279126fbc8ca014a22e9ae5eaf0c98b1e118 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:58:43 +0000 Subject: [PATCH 0275/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 08eefbfff..ee7e933f1 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -19,6 +19,8 @@ Connect to HiveServer2 and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes +Useful for reconciliations between clusters after migrations + Tested on Hive 1.1.0 CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 34431bfdc84ef95e3f25bd9fd409f3882f8cba7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 18:00:36 +0000 Subject: [PATCH 0276/2295] updated README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 142917fad..b69840f56 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,9 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterates all matching tables in all databases (by partition if available) and outputs TSV of table names and row counts - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterates all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs TSV list of tables and their row counts, optionally filtering by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs list of tables containing NULLs columns, optionally filtering by database / table name regex (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 67c2cdd75ff45d7af3fc75dbc97145d1e8d5288e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 18:01:33 +0000 Subject: [PATCH 0277/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b69840f56..51a4c0b38 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ Environment variables are supported for convenience and also to hide credentials - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs TSV list of tables and their row counts, optionally filtering by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs list of tables containing NULLs columns, optionally filtering by database / table name regex (useful for catching data quality or ETL problems) + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs, can filter by database / table name regex (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 9dd8e14117f1180a0ac293c60245403fc452f006 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:37:55 +0000 Subject: [PATCH 0278/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 7dab3781e..9b66f0489 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -22,7 +22,9 @@ Describes each table, constructs a complex query to check each column individually for containing only NULLs, and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching problems with data quality or broken ETL processes +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From 5b66fa8dc48b0c20cdc95b5c303c86567684aac6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:38:00 +0000 Subject: [PATCH 0279/2295] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 05c149456..62e984e64 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -22,7 +22,9 @@ Describes each table, constructs a complex query to check each column individually for containing only NULLs, and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching problems with data quality or broken ETL processes +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From 423ef5e98a444fde933d02a6bbc29d9ecc3a4b3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:39:20 +0000 Subject: [PATCH 0280/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 9b66f0489..ffb1a7dce 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -128,7 +128,7 @@ def check_table_for_nulls(conn, database, table, columns, query): ) ) else: - print('OK: {db}.{table} as {num}/{total} columns with only nulls'\ + print('OK: {db}.{table} has {num}/{total} columns with only nulls'\ .format(db=database, table=table, num=num_cols, total=total_cols)) From 34edb23d60dcb80e3b6e53fd2370b65976c4701a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:50:49 +0000 Subject: [PATCH 0281/2295] quoted all column, db and table names to avoid column names like 'role' breaking queries --- hive_tables_null_columns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index ffb1a7dce..985619ec1 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -65,7 +65,7 @@ __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' class HiveTablesNullColumns(HiveForEachTable): @@ -96,12 +96,12 @@ def execute(self, conn, database, table, query): #column_type = column_row[1] columns.append(column) sum_part = ', '.join( - ['IF(SUM(IF({col} IS NULL, 1, 0)) = COUNT(*), 1, 0) as {col}'.format(col=column) \ + ['IF(SUM(IF(`{col}` IS NULL, 1, 0)) = COUNT(*), 1, 0) as `{col}`'.format(col=column) \ for column in columns] ) - query = "SELECT {sum_part} FROM {db}.{table} WHERE "\ + query = "SELECT {sum_part} FROM `{db}`.`{table}` WHERE `"\ .format(sum_part=sum_part, db=database, table=table) + \ - " IS NULL OR ".join(columns) + " IS NULL" + "` IS NULL OR `".join(columns) + "` IS NULL" self.check_table_for_nulls(conn, database, table, columns, query) @staticmethod From ae109aad324c71567ffe1ecd83a8754d913f1aa0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:56:59 +0000 Subject: [PATCH 0282/2295] quoted all db, table and column references --- hive_foreach_table.py | 4 ++-- hive_schemas_csv.py | 8 ++++---- hive_tables_null_columns.py | 4 ++-- hive_tables_row_counts.py | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 074a2b55d..423c1a78b 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.1' +__version__ = '0.4.2' class HiveForEachTable(HiveImpalaCLI): @@ -146,7 +146,7 @@ def run(self): try: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: log.error(_) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 1dbb3dff3..da6e8e889 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveSchemasCSV(HiveImpalaCLI): @@ -134,7 +134,7 @@ def run(self): with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') for table_row in table_cursor: table = table_row[0] @@ -143,8 +143,8 @@ def run(self): # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) for column_row in column_cursor: column = column_row[0] column_type = column_row[1] diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 985619ec1..c11716b75 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -89,8 +89,8 @@ def execute(self, conn, database, table, query): # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) for column_row in column_cursor: column = column_row[0] #column_type = column_row[1] diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index ee7e933f1..36fa80811 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -125,7 +125,7 @@ def run(self): try: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: log.error(_) @@ -152,9 +152,9 @@ def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) with conn.cursor() as partition_cursor: # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) + partition_cursor.execute('use `{db}`'.format(db=database)) try: - partition_cursor.execute('show partitions {table}'.format(table=table)) + partition_cursor.execute('show partitions `{table}`'.format(table=table)) for partitions_row in partition_cursor: partition_key = partitions_row[0] partition_value = partitions_row[1] @@ -168,7 +168,7 @@ def get_row_counts(self, conn, database, table, partition_regex): self.partition) continue # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + partition_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}` WHERE `{key}`={value}'\ .format(db=database, table=table, key=partition_key, value=partition_value)) for result in partition_cursor: row_count = result[0] @@ -187,9 +187,9 @@ def get_row_counts(self, conn, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", database, table) with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table) # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + table_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table)) for result in table_cursor: row_count = result[0] print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) From 5ba64d63be973a2460f452a2c34835f364423fac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:15:48 +0000 Subject: [PATCH 0283/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index c11716b75..316d5336f 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -24,7 +24,7 @@ Useful for catching problems with data quality or subtle ETL bugs -Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo +Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From 580f749e1b6f0ad0ee8c3513fc32b8b84e620f9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:15:59 +0000 Subject: [PATCH 0284/2295] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 62e984e64..bd2eb0e61 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -24,7 +24,7 @@ Useful for catching problems with data quality or subtle ETL bugs -Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo +Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From a22fbf1a5deea460134d00c9b28b93329b96f5e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:19:48 +0000 Subject: [PATCH 0285/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 316d5336f..0c72c07aa 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -74,7 +74,7 @@ def __init__(self): super(HiveTablesNullColumns, self).__init__() # Python 3.x # super().__init__() - self.query = 'placeholder' # constructed later dynamically per table + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option self.database = None self.table = None #self.partition = None From 8e41d7cade898da4a8fee457f66156f64c7bf000 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:31:45 +0000 Subject: [PATCH 0286/2295] added hive_tables_null_rows.py --- hive_tables_null_rows.py | 106 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 hive_tables_null_rows.py diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py new file mode 100755 index 000000000..60c43e79c --- /dev/null +++ b/hive_tables_null_rows.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count number of rows with only NULLs in all columns +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesNullRows(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesNullRows, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL AND `".join(columns) + "` IS NULL" + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + for result in table_cursor: + count = result[0] + print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + + +if __name__ == '__main__': + HiveTablesNullRows().main() From 1cdb73bcf5d52f08906755c20157fb653d0bc5d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:31:52 +0000 Subject: [PATCH 0287/2295] added impala_tables_null_rows.py --- impala_tables_null_rows.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_null_rows.py diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py new file mode 100755 index 000000000..3da2b5a2a --- /dev/null +++ b/impala_tables_null_rows.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count number of rows with only NULLs in all columns +for each tables in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_null_rows import HiveTablesNullRows +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesNullRows(HiveTablesNullRows): + + def __init__(self): + # Python 2.x + super(ImpalaTablesNullRows, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesNullRows().main() From e9178a0cff24faf15ddb44dfdb18fd46dff5f610 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:34:06 +0000 Subject: [PATCH 0288/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 51a4c0b38..9989973f9 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs, can filter by database / table name regex (useful for catching data quality or ETL problems) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 480120392cd82109d622f637034e05a3a0047ed4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:42:54 +0000 Subject: [PATCH 0289/2295] added hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 106 ++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 hive_tables_row_counts_any_nulls.py diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py new file mode 100755 index 000000000..72ea0c67c --- /dev/null +++ b/hive_tables_row_counts_any_nulls.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count number of rows with NULL in any column +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesRowsWithNulls(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesRowsWithNulls, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL OR `".join(columns) + "` IS NULL" + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + for result in table_cursor: + count = result[0] + print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + + +if __name__ == '__main__': + HiveTablesRowsWithNulls().main() From 23f5d169fdf0a1f7e51e8436105d7297e3538167 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:42:59 +0000 Subject: [PATCH 0290/2295] added impala_tables_row_counts_any_nulls.py --- impala_tables_row_counts_any_nulls.py | 81 +++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_row_counts_any_nulls.py diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py new file mode 100755 index 000000000..5cfbc009d --- /dev/null +++ b/impala_tables_row_counts_any_nulls.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count number of rows with NULL in any column +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_counts_any_nulls import HiveTablesRowsWithNulls +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesRowsWithNulls(HiveTablesRowsWithNulls): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowsWithNulls, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowsWithNulls().main() From 89b24267b6f1c0df963db413763d26402137c929 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:45:57 +0000 Subject: [PATCH 0291/2295] updated README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9989973f9..44d1ffcc2 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,11 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table. Optionally filter by database / table name regex + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 857961a718a7c97cb7e4397a9e6b6e22bca054c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:49:43 +0000 Subject: [PATCH 0292/2295] updated README.md --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 44d1ffcc2..478a95a70 100644 --- a/README.md +++ b/README.md @@ -113,11 +113,12 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table. Optionally filter by database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex + The following programs can all optionally filter by database / table name regex: + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations + - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From a72b47f8ae4ef9baf3af40eaf29d2e6c74867b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:50:03 +0000 Subject: [PATCH 0293/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 478a95a70..c52a0d279 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,9 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + The following programs can all optionally filter by database / table name regex: + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs From e6431a7d1126a0b5a0b994ea03e3ad7327fb1fc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 14:23:09 +0000 Subject: [PATCH 0294/2295] unnested iterations to avoid long running query handles incurring 'impala.error.HiveServer2Error: Invalid query handle' --- hive_foreach_table.py | 70 ++++++++++++++++++++++----------------- hive_tables_row_counts.py | 64 +++++++++++++++++++---------------- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 423c1a78b..9ff0750eb 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' class HiveForEachTable(HiveImpalaCLI): @@ -134,6 +134,9 @@ def run(self): #partition_regex = re.compile(self.partition, re.I) conn = self.connect('default') log.info('querying databases') + # collecting in local list because long time iteration results in + # impala.error.HiveServer2Error: Invalid query handle + databases = [] with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: @@ -141,36 +144,41 @@ def run(self): if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - try: - query = self.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = self.query.format(table=table) - try: - self.execute(conn, database, table, query) - except Exception as _: - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + databases.append(database) + for database in databases: + tables = [] + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + for table in tables: + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise @staticmethod def execute(conn, database, table, query): diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 36fa80811..6e93c6a84 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -63,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class HiveTablesRowCounts(HiveImpalaCLI): @@ -113,6 +113,9 @@ def run(self): partition_regex = re.compile(self.partition, re.I) conn = self.connect('default') log.info('querying databases') + # collecting in local list because long time iteration results in + # impala.error.HiveServer2Error: Invalid query handle + databases = [] with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: @@ -120,33 +123,38 @@ def run(self): if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - try: - self.get_row_counts(conn, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + databases.append(database) + for database in databases: + tables = [] + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + for table in tables: + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From c8a692c1cfe484df6b9ac8a2aa3e9599738cf89b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:40:46 +0000 Subject: [PATCH 0295/2295] updated hdfs_find_replication_factor_1.py --- hdfs_find_replication_factor_1.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index fa80c3586..0a38a9316 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -123,8 +123,7 @@ def run(self): print('', file=sys.stderr) print(file_path) if self.replication_factor: - log.info('setting replication factor to {} on {}'\ - .format(self.replication_factor, file_path)) + log.info('setting replication factor to %s on %s', self.replication_factor, file_path) # returns a generator so must evaluate in order to actually execute # otherwise you find there is no effect on the replication factor for _ in client.setrep([file_path], self.replication_factor, recurse=False): From 64a013c7addb4707568165d511072344acf37157 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:11 +0000 Subject: [PATCH 0296/2295] added database + table counters --- hive_foreach_table.py | 79 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 9ff0750eb..ea10ed5a7 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveForEachTable(HiveImpalaCLI): @@ -137,53 +137,62 @@ def run(self): # collecting in local list because long time iteration results in # impala.error.HiveServer2Error: Invalid query handle databases = [] + database_count = 0 with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] + database_count += 1 if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue databases.append(database) + log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - tables = [] - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - tables.append(table) - for table in tables: - try: - query = self.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = self.query.format(table=table) - try: - self.execute(conn, database, table, query) - except Exception as _: - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + self.process_database(conn, database, table_regex) + + def process_database(self, conn, database, table_regex): + tables = [] + table_count = 0 + log.info("querying tables for database '%s'", database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error('error querying tables for database %s: %s', database, _) + if 'AuthorizationException' in str(_): + return + raise + for table_row in table_cursor: + table = table_row[0] + table_count += 1 + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) + for table in tables: + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise @staticmethod def execute(conn, database, table, query): try: - log.info(" %s.%s - running %s", database, table, query) + log.info(' %s.%s - running %s', database, table, query) with conn.cursor() as query_cursor: # doesn't support parameterized query quoting from dbapi spec query_cursor.execute(query) From b8895386d05dd0cea7ea26d440654447e5dedfdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:16 +0000 Subject: [PATCH 0297/2295] added database + table counters --- hive_schemas_csv.py | 109 ++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index da6e8e889..5fc218aa3 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -61,6 +61,7 @@ import csv import os import sys +import impala srcdir = os.path.abspath(os.path.dirname(__file__)) pylib = os.path.join(srcdir, 'pylib') lib = os.path.join(srcdir, 'lib') @@ -90,6 +91,11 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None + self.table_count = 0 + self.column_count = 0 + self.ignore_errors = False + self.csv_writer = None + self.conn = None def add_options(self): super(HiveSchemasCSV, self).add_options() @@ -101,57 +107,94 @@ def add_options(self): self.add_opt('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV (recommended, default is double quote \'"\')') self.add_opt('-E', '--escapechar', help='Escape char if needed') +# +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# + self.add_opt('-e', '--ignore-errors', action='store_true', + help='Ignore individual table schema listing errors (Impala often has table metadata errors)') def process_options(self): super(HiveSchemasCSV, self).process_options() self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') + self.ignore_errors = self.get_opt('ignore_errors') def run(self): - conn = self.connect('default') + self.conn = self.connect('default') quoting = csv.QUOTE_ALL if self.quotechar == '': quoting = csv.QUOTE_NONE fieldnames = ['database', 'table', 'column', 'type'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=self.delimiter, - quotechar=self.quotechar, - escapechar=self.escapechar, - quoting=quoting, - fieldnames=fieldnames) - csv_writer.writeheader() + self.csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + self.csv_writer.writeheader() log.info('querying databases') - with conn.cursor() as db_cursor: + databases = [] + database_count = 0 + with self.conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - for table_row in table_cursor: - table = table_row[0] - log.info('describing table %s.%s', database, table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use `{}`'.format(database)) - column_cursor.execute('describe `{}`'.format(table)) - for column_row in column_cursor: - column = column_row[0] - column_type = column_row[1] - csv_writer.writerow({'database': database, - 'table': table, - 'column': column, - 'type': column_type}) + database_count += 1 + databases.append(database) + log.info('found %s databases', database_count) + for database in databases: + self.process_database(database) + log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count) + + def process_database(self, database): + log.info('querying tables for database %s', database) + tables = [] + with self.conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + self.table_count += 1 + tables.append(table) + for table in tables: + try: + self.process_table(database, table) + except impala.error.HiveServer2Error as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + def process_table(self, database, table): + log.info('describing table %s.%s', database, table) + with self.conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + column_type = column_row[1] + self.column_count += 1 + self.csv_writer.writerow({'database': database, + 'table': table, + 'column': column, + 'type': column_type}) if __name__ == '__main__': From c066d6c251790e6de98e1e942709b07667eb9011 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:22 +0000 Subject: [PATCH 0298/2295] added database + table counters --- hive_tables_row_counts.py | 71 ++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 6e93c6a84..b9b06a0e8 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -63,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class HiveTablesRowCounts(HiveImpalaCLI): @@ -116,45 +116,54 @@ def run(self): # collecting in local list because long time iteration results in # impala.error.HiveServer2Error: Invalid query handle databases = [] + database_count = 0 with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] + database_count += 1 if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue databases.append(database) + log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - tables = [] - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - tables.append(table) - for table in tables: - try: - self.get_row_counts(conn, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + self.process_database(conn, database, table_regex, partition_regex) + + def process_database(self, conn, database, table_regex, partition_regex): + tables = [] + table_count = 0 + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + return + raise + for table_row in table_cursor: + table = table_row[0] + table_count += 1 + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) + for table in tables: + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From cbe5a865df624e031773f3f133db89b82e9f4fca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:47:41 +0000 Subject: [PATCH 0299/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 5fc218aa3..c47fbad13 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -170,6 +170,7 @@ def process_database(self, database): table = table_row[0] self.table_count += 1 tables.append(table) + log.info("found %s tables in database '%s'", len(tables), database) for table in tables: try: self.process_table(database, table) From 0a4d6cfd3c015b4bb624e6f7adb856601023f3aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:49:12 +0000 Subject: [PATCH 0300/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index c47fbad13..eddef782c 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -188,14 +188,17 @@ def process_table(self, database, table): #column_cursor.execute('describe %(table)s', {'table': table}) column_cursor.execute('use `{}`'.format(database)) column_cursor.execute('describe `{}`'.format(table)) + column_count = 0 for column_row in column_cursor: column = column_row[0] column_type = column_row[1] - self.column_count += 1 + column_count += 1 self.csv_writer.writerow({'database': database, 'table': table, 'column': column, 'type': column_type}) + log.info("found %s columns in table '%s.%s'", column_count, database, table) + self.column_count += column_count if __name__ == '__main__': From fa5db3aef2ace8c3456615be8f7b18368ef046f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:52:01 +0000 Subject: [PATCH 0301/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index ea10ed5a7..245e9bdf5 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -90,6 +90,7 @@ def __init__(self): self.table = None self.partition = None self.ignore_errors = False + self.table_count = 0 def add_options(self): super(HiveForEachTable, self).add_options() @@ -150,6 +151,7 @@ def run(self): log.info('%s/%s databases selected', len(databases), database_count) for database in databases: self.process_database(conn, database, table_regex) + log.info('processed %s databases, %s tables', database_count, self.table_count) def process_database(self, conn, database, table_regex): tables = [] @@ -183,6 +185,7 @@ def process_database(self, conn, database, table_regex): query = self.query.format(table=table) try: self.execute(conn, database, table, query) + self.table_count += 1 except Exception as _: if self.ignore_errors: log.error("database '%s' table '%s': %s", database, table, _) From ad7b1ab6843c1debaadfa62f4ad7fbd88dd13bb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:55:57 +0000 Subject: [PATCH 0302/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index b9b06a0e8..52726e063 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -77,6 +77,7 @@ def __init__(self): self.table = None self.partition = None self.ignore_errors = False + self.table_count = 0 def add_options(self): super(HiveTablesRowCounts, self).add_options() @@ -129,6 +130,7 @@ def run(self): log.info('%s/%s databases selected', len(databases), database_count) for database in databases: self.process_database(conn, database, table_regex, partition_regex) + log.info('processed %s databases, %s tables', database_count, self.table_count) def process_database(self, conn, database, table_regex, partition_regex): tables = [] @@ -157,6 +159,7 @@ def process_database(self, conn, database, table_regex, partition_regex): for table in tables: try: self.get_row_counts(conn, database, table, partition_regex) + self.table_count += 1 except Exception as _: # invalid query handle and similar errors happen at higher level # as they are not query specific, will not be caught here so still error out From 7d91b62a2b3f2138512f76fb7e182ed8da91f871 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:57:29 +0000 Subject: [PATCH 0303/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index eddef782c..9a687152b 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -159,7 +159,7 @@ def run(self): log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count) def process_database(self, database): - log.info('querying tables for database %s', database) + log.info("querying tables for database '%s'", database) tables = [] with self.conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec From 8b031b983a7b99b32e47e57ce60ca90f97659e85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:57:47 +0000 Subject: [PATCH 0304/2295] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 52726e063..d62ecf77c 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -135,7 +135,7 @@ def run(self): def process_database(self, conn, database, table_regex, partition_regex): tables = [] table_count = 0 - log.info('querying tables for database %s', database) + log.info("querying tables for database '%s'", database) with conn.cursor() as table_cursor: try: # doesn't support parameterized query quoting from dbapi spec From 8f20568c52f651e5b72fcb8657ef38901edb2c5d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:58:10 +0000 Subject: [PATCH 0305/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 245e9bdf5..0f783fce4 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -164,7 +164,7 @@ def process_database(self, conn, database, table_regex): table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: - log.error('error querying tables for database %s: %s', database, _) + log.error("error querying tables for database '%s': %s", database, _) if 'AuthorizationException' in str(_): return raise From b71bf376bcd777c8428d86d8a53fd40e64e0d5f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:59:44 +0000 Subject: [PATCH 0306/2295] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 9a687152b..9f4ee016a 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -181,7 +181,7 @@ def process_database(self, database): raise def process_table(self, database, table): - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with self.conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From bc0597e68ecfc144b7f52c6d27854ce320f28a48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:14 +0000 Subject: [PATCH 0307/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 0c72c07aa..3f16e9f5c 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -84,7 +84,7 @@ def __init__(self): def execute(self, conn, database, table, query): sum_part = '' columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 88abb33c82c87bd6ede9f3c520037c85fafb7f29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:23 +0000 Subject: [PATCH 0308/2295] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 60c43e79c..698b72a78 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -80,7 +80,7 @@ def __init__(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 7fccea2201627e3727e5f4791464179fe1c0fff0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:37 +0000 Subject: [PATCH 0309/2295] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 72ea0c67c..21e248f2b 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -80,7 +80,7 @@ def __init__(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From e26f222c51c183bd471da581c5fbb0a34b3327fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:08:03 +0000 Subject: [PATCH 0310/2295] added hive_tables_list.py --- hive_tables_list.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 hive_tables_list.py diff --git a/hive_tables_list.py b/hive_tables_list.py new file mode 100755 index 000000000..7416dd298 --- /dev/null +++ b/hive_tables_list.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and list all databases and tables + +TSV Output format: + +
+ + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class HiveTablesList(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesList, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # here merely to suppress --query CLI option + + def execute(self, conn, database, table, query): + print('{}\t{}'.format(database, table)) + + +if __name__ == '__main__': + HiveTablesList().main() From f21ee01adb5973a033b37fa97bb790adbcfa3b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:08:11 +0000 Subject: [PATCH 0311/2295] added impala_tables_list.py --- impala_tables_list.py | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_list.py diff --git a/impala_tables_list.py b/impala_tables_list.py new file mode 100755 index 000000000..caddf99ec --- /dev/null +++ b/impala_tables_list.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list all databases and tables + +TSV Output format: + +
+ + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_list import HiveTablesList +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ImpalaTablesList(HiveTablesList): + + def __init__(self): + # Python 2.x + super(ImpalaTablesList, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesList().main() From f9588522a1e558a3ab1f7fb88f958818cacf0569 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:11:48 +0000 Subject: [PATCH 0312/2295] updated tested versions --- hive_foreach_table.py | 2 +- hive_schemas_csv.py | 2 +- hive_tables_list.py | 2 +- hive_tables_null_columns.py | 2 +- hive_tables_null_rows.py | 2 +- hive_tables_row_counts.py | 2 +- hive_tables_row_counts_any_nulls.py | 2 +- impala_foreach_table.py | 2 +- impala_schemas_csv.py | 2 +- impala_tables_list.py | 2 +- impala_tables_null_columns.py | 2 +- impala_tables_null_rows.py | 2 +- impala_tables_row_counts.py | 2 +- impala_tables_row_counts_any_nulls.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 0f783fce4..edb15e434 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -31,7 +31,7 @@ hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 9f4ee016a..5daf009d6 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_list.py b/hive_tables_list.py index 7416dd298..b0c4f4702 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -23,7 +23,7 @@
-Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 3f16e9f5c..a78e049f4 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -26,7 +26,7 @@ Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 698b72a78..78c42d15e 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index d62ecf77c..b37059c9b 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -21,7 +21,7 @@ Useful for reconciliations between clusters after migrations -Tested on Hive 1.1.0 CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 21e248f2b..0f7a5a94a 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 49c8e21e0..7b35b0174 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -31,7 +31,7 @@ impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 17ce8987b..d6555b542 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_list.py b/impala_tables_list.py index caddf99ec..feb4e73ca 100755 --- a/impala_tables_list.py +++ b/impala_tables_list.py @@ -23,7 +23,7 @@
-Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index bd2eb0e61..ad968fbc1 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -26,7 +26,7 @@ Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py index 3da2b5a2a..c70484540 100755 --- a/impala_tables_null_rows.py +++ b/impala_tables_null_rows.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 99f2d5b5a..47c104789 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -21,7 +21,7 @@ Useful for reconciliations between clusters after migrations -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py index 5cfbc009d..ab6a6c638 100755 --- a/impala_tables_row_counts_any_nulls.py +++ b/impala_tables_row_counts_any_nulls.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f179178e1d2840e4e47229f1474af83936d2692c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 18:09:34 +0000 Subject: [PATCH 0313/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 7335c0168..966142672 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -488,6 +488,9 @@ dest[140]='ssh -i myKey -N -L 8888::8888 @' src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +src[142]="ERROR: AnalysisException: Failed to load metadata for table: 'myCustomerTable2'" +dest[142]="ERROR: AnalysisException: Failed to load metadata for table: '
'" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 1da475945ce57620c5d65410e5617b73ffa00643 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 18:09:44 +0000 Subject: [PATCH 0314/2295] updated anonymize.py --- anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 106f73952..94beaf3f6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.9' +__version__ = '0.10.10' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -319,7 +319,7 @@ def __init__(self): .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', + 'db4': r'(\s(?:in|of|for)\s+(column|table|database|schema)[\s:]+[\'"])[^\'"]+', 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ From 9c693e1c5141aa3ac4809ebc20edc95f43ad49d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Jan 2020 14:25:40 +0000 Subject: [PATCH 0315/2295] added quoting and connect to each db --- hive_foreach_table.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index edb15e434..4ef224290 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.1' +__version__ = '0.5.2' class HiveForEachTable(HiveImpalaCLI): @@ -150,13 +150,14 @@ def run(self): databases.append(database) log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - self.process_database(conn, database, table_regex) + self.process_database(database, table_regex) log.info('processed %s databases, %s tables', database_count, self.table_count) - def process_database(self, conn, database, table_regex): + def process_database(self, database, table_regex): tables = [] table_count = 0 log.info("querying tables for database '%s'", database) + conn = self.connect(database) with conn.cursor() as table_cursor: try: # doesn't support parameterized query quoting from dbapi spec @@ -179,10 +180,13 @@ def process_database(self, conn, database, table_regex): log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) for table in tables: try: - query = self.query.format(db=database, table=table) + query = self.query.format(db='`{}`'.format(database), + table='`{}`'.format(table)) except KeyError as _: if _ == 'db': - query = self.query.format(table=table) + query = self.query.format(table='`{}`'.format(table)) + else: + raise try: self.execute(conn, database, table, query) self.table_count += 1 From 0e8f4c6615153c8f660c1f3aa19710a95c5a3d7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:50:32 +0000 Subject: [PATCH 0316/2295] added hive_tables_locations.py --- hive_tables_locations.py | 90 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 hive_tables_locations.py diff --git a/hive_tables_locations.py b/hive_tables_locations.py new file mode 100755 index 000000000..cadccf9e2 --- /dev/null +++ b/hive_tables_locations.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesLocations(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'describe formatted {table}' + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + log.info("describing table '%s.%s'", database, table) + location = 'UNKNOWN' + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + #table_cursor.execute('describe %(table)s', {'table': table}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute(query.format(table=table)) + for row in table_cursor: + if 'Location' in row[0]: + location = row[1] + break + print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + + +if __name__ == '__main__': + HiveTablesLocations().main() From c39c8f803aa3e3e18b2686dd771a56c155f434f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:50:37 +0000 Subject: [PATCH 0317/2295] added impala_tables_locations.py --- impala_tables_locations.py | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_locations.py diff --git a/impala_tables_locations.py b/impala_tables_locations.py new file mode 100755 index 000000000..1182164c6 --- /dev/null +++ b/impala_tables_locations.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_locations import HiveTablesLocations +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesLocations(HiveTablesLocations): + + def __init__(self): + # Python 2.x + super(ImpalaTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesLocations().main() From 4823788f6f482a20fbc9a6eb6ec80882990fdeb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:58:40 +0000 Subject: [PATCH 0318/2295] added hive_tables_metadata.py --- hive_tables_metadata.py | 111 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 hive_tables_metadata.py diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py new file mode 100755 index 000000000..c2e20552c --- /dev/null +++ b/hive_tables_metadata.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') +for all tables in all databases, or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesMetadata(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'describe formatted {table}' + self.field = None + + def add_options(self): + # Python 2.x + super(HiveTablesMetadata, self).add_options() + # Python 3.x + # super().__init__() + if self.field is None: + self.add_opt('-f', '--field', help='Table DDL metadata field to return for each table (required)') + + def process_options(self): + # Python 2.x + super(HiveTablesMetadata, self).process_options() + # Python 3.x + # super().__init__() + field = self.get_opt('field') + if not field: + self.usage('--field not specified') + validate_regex(field, 'field') + self.field = re.compile(field) + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + log.info("describing table '%s.%s'", database, table) + location = 'UNKNOWN' + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + #table_cursor.execute('describe %(table)s', {'table': table}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute(query.format(table=table)) + for row in table_cursor: + if self.field.search(row[0]): + location = row[1] + break + print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + + +if __name__ == '__main__': + HiveTablesMetadata().main() From 16500c28e2f0954841bcd89ea9d478aa290d0c0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:58:43 +0000 Subject: [PATCH 0319/2295] added impala_tables_metadata.py --- impala_tables_metadata.py | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_metadata.py diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py new file mode 100755 index 000000000..8182f74e1 --- /dev/null +++ b/impala_tables_metadata.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_metadata import HiveTablesMetadata +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesMetadata(HiveTablesMetadata): + + def __init__(self): + # Python 2.x + super(ImpalaTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesMetadata().main() From 57cedbe51323f2623d1a6a7d465b812bf890f573 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:01:17 +0000 Subject: [PATCH 0320/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index c2e20552c..6de7888cb 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -84,7 +84,8 @@ def process_options(self): super(HiveTablesMetadata, self).process_options() # Python 3.x # super().__init__() - field = self.get_opt('field') + if self.field is None: + field = self.get_opt('field') if not field: self.usage('--field not specified') validate_regex(field, 'field') From ef6473e124eb5ed0e2ad09cf7e79573a7d1c71d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:01:28 +0000 Subject: [PATCH 0321/2295] updated hive_tables_locations.py --- hive_tables_locations.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/hive_tables_locations.py b/hive_tables_locations.py index cadccf9e2..af17aaed8 100755 --- a/hive_tables_locations.py +++ b/hive_tables_locations.py @@ -48,8 +48,7 @@ sys.path.append(pylib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from hive_foreach_table import HiveForEachTable + from hive_tables_metadata import HiveTablesMetadata except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -60,30 +59,14 @@ __version__ = '0.5.0' -class HiveTablesLocations(HiveForEachTable): +class HiveTablesLocations(HiveTablesMetadata): def __init__(self): # Python 2.x super(HiveTablesLocations, self).__init__() # Python 3.x # super().__init__() - self.query = 'describe formatted {table}' - - # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): - log.info("describing table '%s.%s'", database, table) - location = 'UNKNOWN' - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - #table_cursor.execute('describe %(table)s', {'table': table}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute(query.format(table=table)) - for row in table_cursor: - if 'Location' in row[0]: - location = row[1] - break - print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + self.field = 'Location' if __name__ == '__main__': From 46d3954fc58d8f0324d0565b4bb5a5c2b5c81355 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:02:47 +0000 Subject: [PATCH 0322/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 6de7888cb..ccf66fc2a 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -85,11 +85,11 @@ def process_options(self): # Python 3.x # super().__init__() if self.field is None: - field = self.get_opt('field') - if not field: + self.field = self.get_opt('field') + if not self.field: self.usage('--field not specified') - validate_regex(field, 'field') - self.field = re.compile(field) + validate_regex(self.field, 'field') + self.field = re.compile(self.field) # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): From e48535cf3385a13cac643d15b3d7353c77b8c6db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:05:52 +0000 Subject: [PATCH 0323/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c52a0d279..342ceb725 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table + - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From eeab4a654786f323a574f65107d48c4e5a36acd4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:38:20 +0000 Subject: [PATCH 0324/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1e33eaec4..dac327ef0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1e33eaec4117818ee32e3644e101a715f5a66de0 +Subproject commit dac327ef02f570cd08b426a4e9e90a97d0171357 From 23fd6c99c25b4c3f3711d57c4246053834cff9cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:38:21 +0000 Subject: [PATCH 0325/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4dcc62e62..ade8241d3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4dcc62e62cbc569a0422a1f57be20ceef32054d0 +Subproject commit ade8241d30c3f7049c65d6de6fb6df1dad273b83 From dee2eb59b10c94262a5332bbeba91f969c2cb8c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0326/2295] added .github/workflows/ci_alpine.yaml --- .github/workflows/ci_alpine.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_alpine.yaml diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/ci_alpine.yaml new file mode 100644 index 000000000..1a10eb7b2 --- /dev/null +++ b/.github/workflows/ci_alpine.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Alpine + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: harisekhon/alpine-dev + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 86d69a8673347235eb91cd99704ec44cca1fe921 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0327/2295] added .github/workflows/ci_centos.yaml --- .github/workflows/ci_centos.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_centos.yaml diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/ci_centos.yaml new file mode 100644 index 000000000..4aba51533 --- /dev/null +++ b/.github/workflows/ci_centos.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: harisekhon/centos-dev + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 05af7465ffca413357c213ecc554c89c73cca619 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0328/2295] added .github/workflows/ci_mac.yaml --- .github/workflows/ci_mac.yaml | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/ci_mac.yaml diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/ci_mac.yaml new file mode 100644 index 000000000..8994825ff --- /dev/null +++ b/.github/workflows/ci_mac.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:39:47 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Mac + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From a2592f847dfa586c03fba9d4c1489faf7c9511ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0329/2295] added .github/workflows/ci_python_2.7.yaml --- .github/workflows/ci_python_2.7.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/ci_python_2.7.yaml diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/ci_python_2.7.yaml new file mode 100644 index 000000000..8a5fea1a4 --- /dev/null +++ b/.github/workflows/ci_python_2.7.yaml @@ -0,0 +1,49 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 2.7 + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [2.7] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From 7fed31e0b7e75dc9d6807b81176e0da2aef598a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0330/2295] added .github/workflows/ci_python_3.6.yaml --- .github/workflows/ci_python_3.6.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/ci_python_3.6.yaml diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/ci_python_3.6.yaml new file mode 100644 index 000000000..2822288a0 --- /dev/null +++ b/.github/workflows/ci_python_3.6.yaml @@ -0,0 +1,49 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.6 + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.6] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From c5a578c33f5035f57870c6c8b7f476ebf2d3322e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0331/2295] added .github/workflows/ci_ubuntu.yaml --- .github/workflows/ci_ubuntu.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_ubuntu.yaml diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ci_ubuntu.yaml new file mode 100644 index 000000000..c48b25b1b --- /dev/null +++ b/.github/workflows/ci_ubuntu.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-16.04] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From 62040b10672dac2d5c3bd641e7bd7996dfc55dca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:41:05 +0000 Subject: [PATCH 0332/2295] updated README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 342ceb725..c246cdde3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) +[![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) +[![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) +[![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) +[![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) +[![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) + ### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. From b74d35865d966c482cc5532025229fe2d3d5329c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:15 +0000 Subject: [PATCH 0333/2295] updated ci_alpine.yaml --- .github/workflows/ci_alpine.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/ci_alpine.yaml index 1a10eb7b2..1b5a0f43c 100644 --- a/.github/workflows/ci_alpine.yaml +++ b/.github/workflows/ci_alpine.yaml @@ -13,8 +13,8 @@ name: CI Alpine -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 72ed22931af6a916a1cb05a0e9cddd1f50e259f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:25 +0000 Subject: [PATCH 0334/2295] updated ci_centos.yaml --- .github/workflows/ci_centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/ci_centos.yaml index 4aba51533..1d8071032 100644 --- a/.github/workflows/ci_centos.yaml +++ b/.github/workflows/ci_centos.yaml @@ -13,8 +13,8 @@ name: CI CentOS -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 815089e0e4ec08d8033b2a2124b90094cb579a9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:31 +0000 Subject: [PATCH 0335/2295] updated ci_mac.yaml --- .github/workflows/ci_mac.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/ci_mac.yaml index 8994825ff..36f71e43d 100644 --- a/.github/workflows/ci_mac.yaml +++ b/.github/workflows/ci_mac.yaml @@ -13,8 +13,8 @@ name: CI Mac -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 00742a66a26ad17a87735160eef6417add5b974d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:37 +0000 Subject: [PATCH 0336/2295] updated ci_python_2.7.yaml --- .github/workflows/ci_python_2.7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/ci_python_2.7.yaml index 8a5fea1a4..e9489be32 100644 --- a/.github/workflows/ci_python_2.7.yaml +++ b/.github/workflows/ci_python_2.7.yaml @@ -13,8 +13,8 @@ name: CI Python 2.7 -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 9f2b8e30fde18b67e59d9b470a9db28d5868c047 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:43 +0000 Subject: [PATCH 0337/2295] updated ci_python_3.6.yaml --- .github/workflows/ci_python_3.6.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/ci_python_3.6.yaml index 2822288a0..16932c744 100644 --- a/.github/workflows/ci_python_3.6.yaml +++ b/.github/workflows/ci_python_3.6.yaml @@ -13,8 +13,8 @@ name: CI Python 3.6 -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From af6adf58c9be8206741013a65c37f054cde2d6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:50 +0000 Subject: [PATCH 0338/2295] updated ci_ubuntu.yaml --- .github/workflows/ci_ubuntu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ci_ubuntu.yaml index c48b25b1b..a919dbf9d 100644 --- a/.github/workflows/ci_ubuntu.yaml +++ b/.github/workflows/ci_ubuntu.yaml @@ -13,8 +13,8 @@ name: CI Ubuntu -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From f7234f960292b9d4e7c23f1f3835ab7117710c47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 23:37:08 +0000 Subject: [PATCH 0339/2295] updated all.sh --- tests/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index 18dddfc25..7aa1f8c75 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; bash-tools/check_all.sh -tests/test_yamllint.sh +#tests/test_yamllint.sh # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME exit 0 From 489cf9095736001341a4fdf41944cabf0a3381c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 09:57:22 +0000 Subject: [PATCH 0340/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c246cdde3..5fadb6284 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) + +[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) From 9ab3b5493e7a3ba2af37bc51313b351c42a7157c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:18:57 +0000 Subject: [PATCH 0341/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index dac327ef0..13e8bf963 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit dac327ef02f570cd08b426a4e9e90a97d0171357 +Subproject commit 13e8bf96340419bd7ec9d1078bba9cf1226105f4 From 8389af956d100ecaf4b3e2a09ab0d787cd84d961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:18:57 +0000 Subject: [PATCH 0342/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ade8241d3..fed3bae88 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ade8241d30c3f7049c65d6de6fb6df1dad273b83 +Subproject commit fed3bae883262763f5587a9154b4324557d5ff4a From e645990ce415172717b276e80e35239d5a057385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:19:26 +0000 Subject: [PATCH 0343/2295] updated Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index f77f6f741..1b19d053d 100755 --- a/Makefile +++ b/Makefile @@ -51,6 +51,11 @@ build: @echo DevOps Python Tools Build @echo ========================= + type -P python + python -V + + pip -V + $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python From 39daf99871e1c5003469747e95adc08e6e648486 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:39:45 +0000 Subject: [PATCH 0344/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fed3bae88..e5c3259a1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fed3bae883262763f5587a9154b4324557d5ff4a +Subproject commit e5c3259a18e3d2439306d7b431e7a57aefe4d83d From fd8a41a0f6958cbff00de239089b2ba805f13100 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:15:49 +0000 Subject: [PATCH 0345/2295] updated Makefile --- Makefile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1b19d053d..ea6b18d4a 100755 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ # =================== +# would fail bootstrapping on Alpine +#SHELL := /usr/bin/env bash + ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif @@ -51,10 +54,11 @@ build: @echo DevOps Python Tools Build @echo ========================= - type -P python - python -V - - pip -V + # executing in sh where type is not available + #type -P python + which python || : + python -V || : + pip -V || : $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi @@ -63,6 +67,12 @@ build: if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi $(MAKE) python + # executing in sh where type is not available + #type -P python + which python + python -V + pip -V + .PHONY: init init: git submodule update --init --recursive From 9e4de76ac9e70af9cbd2e5e3b856c61bf5e13435 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:16:00 +0000 Subject: [PATCH 0346/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 13e8bf963..a9a3d77bc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 13e8bf96340419bd7ec9d1078bba9cf1226105f4 +Subproject commit a9a3d77bcfbd289454cfe5bffbae104692707ca9 From 2945a1cf3a1c00b115f15e85667ba2ce4d3c7cf4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:16:00 +0000 Subject: [PATCH 0347/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e5c3259a1..dab1fcb73 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e5c3259a18e3d2439306d7b431e7a57aefe4d83d +Subproject commit dab1fcb739156831ab71d8e07dd55d92884423ae From 64c7517f58446cf738b7fce32329446acc7e1f95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 14:39:37 +0000 Subject: [PATCH 0348/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a9a3d77bc..b42c6835d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a9a3d77bcfbd289454cfe5bffbae104692707ca9 +Subproject commit b42c6835df3b15c2eafbb13f6356135ebad3a426 From 84597243f3c4b748757cef575a7f97233b7d6d6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 14:39:38 +0000 Subject: [PATCH 0349/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dab1fcb73..5ee1fa9fb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dab1fcb739156831ab71d8e07dd55d92884423ae +Subproject commit 5ee1fa9fb594ebe05693e1099c99fb905a5e0e1c From 9f6be2e987d64e2db3e8420790bfaa9911c467ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 15:09:53 +0000 Subject: [PATCH 0350/2295] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ea6b18d4a..186c1928b 100755 --- a/Makefile +++ b/Makefile @@ -100,7 +100,8 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + NO_FAIL=1 bash-tools/python_pip_install.sh "avro-python3" @# for impyla @#$(SUDO_PIP) pip install --upgrade setuptools || : From 910cca2f66e4ffe2ce130ed57c06836d29d8fb88 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 15:48:16 +0000 Subject: [PATCH 0351/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 186c1928b..066ba0940 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: # Python >= 3.4 - try but accept failure in case we're not on the right version of Python #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi - NO_FAIL=1 bash-tools/python_pip_install.sh "avro-python3" + bash-tools/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) pip install --upgrade setuptools || : From 957c65e9c24505890a6e37653ef66035711eb43d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:41 +0000 Subject: [PATCH 0352/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b42c6835d..fb1dd428f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b42c6835df3b15c2eafbb13f6356135ebad3a426 +Subproject commit fb1dd428fac0ae1d284793bee116f8a413970104 From 81deb21b1a0227e5e97e614028d29ea86ddd4f81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:41 +0000 Subject: [PATCH 0353/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5ee1fa9fb..e91e70074 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5ee1fa9fb594ebe05693e1099c99fb905a5e0e1c +Subproject commit e91e700743944926e8aed036a8808348b015ea1f From 755840dc7c47a8b2636f29879715654e0e14c2b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:50 +0000 Subject: [PATCH 0354/2295] renamed ci_alpine.yaml to alpine.yaml --- .github/workflows/{ci_alpine.yaml => alpine.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_alpine.yaml => alpine.yaml} (100%) diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/alpine.yaml similarity index 100% rename from .github/workflows/ci_alpine.yaml rename to .github/workflows/alpine.yaml From 0285832e7dba620d6017431b79927274ff6bebaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:50 +0000 Subject: [PATCH 0355/2295] renamed ci_centos.yaml to centos.yaml --- .github/workflows/{ci_centos.yaml => centos.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_centos.yaml => centos.yaml} (100%) diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/centos.yaml similarity index 100% rename from .github/workflows/ci_centos.yaml rename to .github/workflows/centos.yaml From 1b4d3dd168ebc85523a8246e6b38fbe3a9c7df7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:51 +0000 Subject: [PATCH 0356/2295] renamed ci_mac.yaml to mac.yaml --- .github/workflows/{ci_mac.yaml => mac.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_mac.yaml => mac.yaml} (100%) diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/mac.yaml similarity index 100% rename from .github/workflows/ci_mac.yaml rename to .github/workflows/mac.yaml From edc95ea402ff0381dcba4e04f99976a57b5f87d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:51 +0000 Subject: [PATCH 0357/2295] renamed ci_python_2.7.yaml to python_2.7.yaml --- .github/workflows/{ci_python_2.7.yaml => python_2.7.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_python_2.7.yaml => python_2.7.yaml} (100%) diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/python_2.7.yaml similarity index 100% rename from .github/workflows/ci_python_2.7.yaml rename to .github/workflows/python_2.7.yaml From 264271d302ecccd0ed62e6fb0d042cd640ffab94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:52 +0000 Subject: [PATCH 0358/2295] renamed ci_python_3.6.yaml to python_3.6.yaml --- .github/workflows/{ci_python_3.6.yaml => python_3.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_python_3.6.yaml => python_3.6.yaml} (100%) diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/python_3.6.yaml similarity index 100% rename from .github/workflows/ci_python_3.6.yaml rename to .github/workflows/python_3.6.yaml From fbd0048d920e96c37d0b8038798f6f2b0963701a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:52 +0000 Subject: [PATCH 0359/2295] renamed ci_ubuntu.yaml to ubuntu.yaml --- .github/workflows/{ci_ubuntu.yaml => ubuntu.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_ubuntu.yaml => ubuntu.yaml} (100%) diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ubuntu.yaml similarity index 100% rename from .github/workflows/ci_ubuntu.yaml rename to .github/workflows/ubuntu.yaml From 39a29c8f334ee476b71116238eb5e4a5226b14b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:25:01 +0000 Subject: [PATCH 0360/2295] renamed python_2.7.yaml to python2.yaml --- .github/workflows/{python_2.7.yaml => python2.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python_2.7.yaml => python2.yaml} (100%) diff --git a/.github/workflows/python_2.7.yaml b/.github/workflows/python2.yaml similarity index 100% rename from .github/workflows/python_2.7.yaml rename to .github/workflows/python2.yaml From 60cb4128543949e055eb052b4d3d050a106f3677 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:25:06 +0000 Subject: [PATCH 0361/2295] renamed python_3.6.yaml to python3.yaml --- .github/workflows/{python_3.6.yaml => python3.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python_3.6.yaml => python3.yaml} (100%) diff --git a/.github/workflows/python_3.6.yaml b/.github/workflows/python3.yaml similarity index 100% rename from .github/workflows/python_3.6.yaml rename to .github/workflows/python3.yaml From a640851fbf5cb8f699b425b0f8b43f32ab5a58a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:28:12 +0000 Subject: [PATCH 0362/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 1b5a0f43c..54663fb45 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -29,7 +29,7 @@ jobs: #name: build timeout-minutes: 10 runs-on: ubuntu-latest - container: harisekhon/alpine-dev + container: alpine steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From 564e49e7ce1aa2cc41fef3446020e1a962940d10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:28:19 +0000 Subject: [PATCH 0363/2295] updated centos.yaml --- .github/workflows/centos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 1d8071032..ff6509ac5 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -29,7 +29,7 @@ jobs: #name: build timeout-minutes: 10 runs-on: ubuntu-latest - container: harisekhon/centos-dev + container: centos steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From ee5c0d4822a20445e2aa3c914336135a17d6cbde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:44:42 +0000 Subject: [PATCH 0364/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 54663fb45..967ca2840 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,6 +38,7 @@ jobs: ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release + apk add --no-cache git make && cd /tmp && git clone https://github.com/harisekhon/devops-python-tools && cd devops-python-tools && From 7798e46f73a758be2d25738108744245670d6b88 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:46:56 +0000 Subject: [PATCH 0365/2295] updated centos.yaml --- .github/workflows/centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ff6509ac5..44b4759d1 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,6 +38,7 @@ jobs: ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release + yum install -y git make && cd /tmp && git clone https://github.com/harisekhon/devops-python-tools && cd devops-python-tools && From 2c8efb22b2d814baacfa6fd52c3cb452272bce07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:51:41 +0000 Subject: [PATCH 0366/2295] added .github/workflows/centos6.yaml --- .github/workflows/centos6.yaml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/centos6.yaml diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml new file mode 100644 index 000000000..3fc59deba --- /dev/null +++ b/.github/workflows/centos6.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 6 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:6 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make && + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 112b87f85a39d0b2ef5b43609404191e178851cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:52:01 +0000 Subject: [PATCH 0367/2295] added .github/workflows/centos7.yaml --- .github/workflows/centos7.yaml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/centos7.yaml diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml new file mode 100644 index 000000000..853d33df3 --- /dev/null +++ b/.github/workflows/centos7.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 7 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:7 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make && + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 661a8e5453e7d8e2a363f3c3e9ed6cc966e534ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 20:51:17 +0000 Subject: [PATCH 0368/2295] updated rpm-packages.txt --- setup/rpm-packages.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index 281cfb022..cf3109c39 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -22,7 +22,7 @@ curl iputils # for htpasswd for docker registry authenticated checks -https-tools +httpd-tools # needed to build pyhs2 # libgsasl-devel saslwrapper-devel From fde1f6695e752ef3cced32c5f461b7defcfdcc9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:10 +0000 Subject: [PATCH 0369/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 967ca2840..9b188de5a 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - apk add --no-cache git make && + apk add --no-cache git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From f3d23a006bbe9f1aa61992f31033af6cadcfb442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:27 +0000 Subject: [PATCH 0370/2295] updated centos.yaml --- .github/workflows/centos.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 44b4759d1..daf5d8d57 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From 65f44e2caa1628e61c982a3f9bc1fec7b06dfde8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:37 +0000 Subject: [PATCH 0371/2295] updated centos6.yaml --- .github/workflows/centos6.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index 3fc59deba..5856a69d4 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From cacc9e4440dc67384a3d518a105ff61587ff596a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:49 +0000 Subject: [PATCH 0372/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 853d33df3..f92b232d4 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From cc369e4176c1ab2007c3575e6bcf0d7d3e330ac1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:11 +0000 Subject: [PATCH 0373/2295] added centos8.yaml --- .github/workflows/centos8.yaml | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/centos8.yaml diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml new file mode 100644 index 000000000..2016b60d6 --- /dev/null +++ b/.github/workflows/centos8.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 8 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:8 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make + - name: git clone + run: | + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From aafdeb56f4dcfab409a897028ac322db9ab6f1b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:24 +0000 Subject: [PATCH 0374/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index a919dbf9d..5b419ce79 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -40,5 +40,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: build & test - run: make build test + - name: build + run: make + - name: test + run: make test From c32147809deacb5aeac978225aaee387dd57030f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:34 +0000 Subject: [PATCH 0375/2295] updated mac.yaml --- .github/workflows/mac.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 36f71e43d..db5e0d0da 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,5 +37,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: build & test - run: make build test + - name: build + run: make + - name: test + run: make test From e89bed8072e12f6651dc0b0916b9dd9230376c56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 8 Feb 2020 09:20:11 +0000 Subject: [PATCH 0376/2295] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5fadb6284..9185ade80 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Hari Sekhon - DevOps Python Tools [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) +[![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) +[![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) +[![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From aa3767659dc2b40c52ec7eff16a034ec5bbc264d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:52:54 +0000 Subject: [PATCH 0377/2295] updated rpm-packages-optional.txt --- setup/rpm-packages-optional.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt index f94b6019d..11e757a6f 100644 --- a/setup/rpm-packages-optional.txt +++ b/setup/rpm-packages-optional.txt @@ -14,3 +14,8 @@ # ============================================================================ # yamllint + +# CentOS <= 7 +snappy-devel +# CentOS 8 +csnappy-devel From 1a6b3aaf49c182fcd2f8ec22dbbc061ed08360dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:53:17 +0000 Subject: [PATCH 0378/2295] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index a0015d809..c812c26ed 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -18,5 +18,7 @@ gcc-c++ cyrus-sasl-devel krb5-devel openssl-devel + +# moved to optional to account for changed package names on CentOS 8 # needed to build python-snappy for avro module -snappy-devel +#snappy-devel From a6140320ddc6be4f2dfcdfc4860675797ad095b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:57:44 +0000 Subject: [PATCH 0379/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fb1dd428f..8bfc4536f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fb1dd428fac0ae1d284793bee116f8a413970104 +Subproject commit 8bfc4536f6bf60c2e73793f266a7e509b60a3b05 From fa3b4f9d8f44da2dbabd0455da0ee2fd0324218b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:57:44 +0000 Subject: [PATCH 0380/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e91e70074..de5d01494 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e91e700743944926e8aed036a8808348b015ea1f +Subproject commit de5d01494f4ba302020254820aed850ff71e0778 From 288d17808971029be0c93e2532a8dc9bc453700b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 10:13:38 +0000 Subject: [PATCH 0381/2295] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index c812c26ed..6dd8fa477 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel +openldap-devel openssl-devel # moved to optional to account for changed package names on CentOS 8 From 0ba717d8b2325ecc524c80096e8883a61ce5b903 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:24 +0000 Subject: [PATCH 0382/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8bfc4536f..1b02ade81 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8bfc4536f6bf60c2e73793f266a7e509b60a3b05 +Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 From 8ccf3fb407916312d2a540acae323c541e1dbf97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:24 +0000 Subject: [PATCH 0383/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index de5d01494..93798d25a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit de5d01494f4ba302020254820aed850ff71e0778 +Subproject commit 93798d25afc89abff39dc42c411356c7884e2f19 From c8403d408fbb21165cb8c727adfa32a62f0f7ee0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:39 +0000 Subject: [PATCH 0384/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b02ade81..05de967ac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 +Subproject commit 05de967ac225d60d9973c827c72674fd8aca7f20 From 42bdb181b22090145d17dfdc7ff11eb273fd5619 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 10:40:44 +0000 Subject: [PATCH 0385/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93798d25a..271e16ac5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93798d25afc89abff39dc42c411356c7884e2f19 +Subproject commit 271e16ac5d0b2d0916bd11cf3541e55db23c1f3e From 963a89e466cd79c3cf70be6f4a9c3a2d0b018ea0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 10:51:18 +0000 Subject: [PATCH 0386/2295] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 066ba0940..ef9f2ce1b 100755 --- a/Makefile +++ b/Makefile @@ -54,8 +54,8 @@ build: @echo DevOps Python Tools Build @echo ========================= - # executing in sh where type is not available - #type -P python + @# executing in sh where type is not available + @#type -P python which python || : python -V || : pip -V || : From 21b468c8fe7715b09c7256b75c65d709e4c7a472 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:13:33 +0000 Subject: [PATCH 0387/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 271e16ac5..379794ebf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 271e16ac5d0b2d0916bd11cf3541e55db23c1f3e +Subproject commit 379794ebf7510100cf07a3d7af5b4127181738d1 From 47e6d69567d7740d58493c043d1d09ec6c97bd80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:31 +0000 Subject: [PATCH 0388/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 05de967ac..91a889ad7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 05de967ac225d60d9973c827c72674fd8aca7f20 +Subproject commit 91a889ad7df753e49d1e8556ede183a11b77646c From 72ede886b74840c92afb8107361d688b083fb80f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:32 +0000 Subject: [PATCH 0389/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 379794ebf..5fd874ae1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 379794ebf7510100cf07a3d7af5b4127181738d1 +Subproject commit 5fd874ae1a2891fdfdcbc7c22d2608ce19b39a6d From 3bb834b12a0510862cede283ffc14c96ee16bf13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:47 +0000 Subject: [PATCH 0390/2295] updated test_welcome.sh --- tests/test_welcome.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 1be838108..9deb7ad8a 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -20,6 +20,11 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; # shellcheck disable=SC1091 -#. ./tests/utils.sh +. ./tests/utils.sh + +# Fedora doesn't have /var/log/wtmp +if ! [ -f /var/log/wtmp ]; then + $sudo touch /var/log/wtmp || : +fi ./welcome.py "$@" From f7ce2417a1557277c87daaac0fd8b0d8aaad708a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:43:29 +0000 Subject: [PATCH 0391/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 91a889ad7..d117e0672 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 91a889ad7df753e49d1e8556ede183a11b77646c +Subproject commit d117e067235a8078cd93b0b4d9425482e2c78150 From 53e4e10b3c5cd659b58a09429b23f77669d22451 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:43:30 +0000 Subject: [PATCH 0392/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5fd874ae1..6eef2ab50 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5fd874ae1a2891fdfdcbc7c22d2608ce19b39a6d +Subproject commit 6eef2ab5001d5ff66cecb3cbd70957b5b841e826 From 4b649cbfe4531ecc26541d2f2270290a21047ddd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 14:09:54 +0000 Subject: [PATCH 0393/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d117e0672..a83041be4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d117e067235a8078cd93b0b4d9425482e2c78150 +Subproject commit a83041be449870e9c98ade290ed837fdbae3e6e5 From 4d2fd5e9405fb29a1cb955156a9aea713e366c13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 14:09:54 +0000 Subject: [PATCH 0394/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6eef2ab50..9bf26312e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6eef2ab5001d5ff66cecb3cbd70957b5b841e826 +Subproject commit 9bf26312e298b90bab29eadfb5d2238778dbbe4a From 9f8ad47242b654a952b66aaa5e047858f9be6667 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:13:38 +0000 Subject: [PATCH 0395/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a83041be4..c8bfc3bab 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a83041be449870e9c98ade290ed837fdbae3e6e5 +Subproject commit c8bfc3bab2d1625eb10d5fa8a37c5e908bc6d8bc From ebee286fd42e8da3dce1c31c389ae5a5d5a35aa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:13:39 +0000 Subject: [PATCH 0396/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9bf26312e..6d460ffcb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9bf26312e298b90bab29eadfb5d2238778dbbe4a +Subproject commit 6d460ffcb0d520ccf1b07d0e0a4762fff45da61d From f701b75d41c1d1e73367e83d68e83132d4c17b93 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:15:25 +0000 Subject: [PATCH 0397/2295] updated rpm-packages.txt --- setup/rpm-packages.txt | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index cf3109c39..cf5f0222b 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -15,12 +15,6 @@ java -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# for ping mode of find_active_server.py -iputils - # for htpasswd for docker registry authenticated checks httpd-tools @@ -28,8 +22,17 @@ httpd-tools # libgsasl-devel saslwrapper-devel #cyrus-sasl-devel +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# for ping mode of find_active_server.py +#iputils + # for anonymize_parallel.sh -parallel +#parallel -unzip -zip +#unzip +#zip From e886aa4a9f9d8ddb2b8f7e77c02c5d8de70a950a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:17:27 +0000 Subject: [PATCH 0398/2295] updated deb-packages.txt --- setup/deb-packages.txt | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index c605fdd27..d7dc245df 100644 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -13,22 +13,9 @@ # Deb Package Requirements # ============================================================================ # -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# either of these should do to ensure ping command is present for find_active_server.py's ping mode -iputils-ping -#inetutils-ping - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - -# needed for serf test's 'uptime' command -procps - # installs 343MB of dependencies - install by hand if needed #ffmpeg @@ -41,5 +28,21 @@ procps #which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk -zip -unzip +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# either of these should do to ensure ping command is present for find_active_server.py's ping mode +#iputils-ping +##inetutils-ping + +# for anonymize_parallel.sh +#parallel + +# needed for serf test's 'uptime' command +#procps + +#zip +#unzip From 94bc98974794dc15fbf5b97a814f11e4b0b217b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:18:55 +0000 Subject: [PATCH 0399/2295] updated apk-packages.txt --- setup/apk-packages.txt | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index fc6956249..bccca7733 100644 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -13,24 +13,27 @@ # Alpine Package Requirements # ============================================================================ # -# full mktemp needed for json tests -# full split needed for anonymize_parallel.sh -coreutils - -grep - -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - #which java || $(SUDO) apk add openjdk8-jre-base # Spark Java Py4J gets java linking error without this #if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi -zip +# ===================================== +# installed by bash-tools submodule now + +# full mktemp needed for json tests +# full split needed for anonymize_parallel.sh +#coreutils + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +#grep + +# for anonymize_parallel.sh +#parallel + +#zip From c668b3cb6cd905f60e9c35700a5ad22ee8fa0ed5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:19:29 +0000 Subject: [PATCH 0400/2295] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index f8738775b..67e6f895d 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -15,4 +15,6 @@ openldap-dev snappy-dev -unzip + +# installed by bash-tools submodule now +#unzip From d667dbd4661bbbe2f8aa367914feb21db7adfca8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:19:56 +0000 Subject: [PATCH 0401/2295] updated brew-packages.txt --- setup/brew-packages.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index fa25317ba..03a0a78a5 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,8 +14,9 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # +# installed by bash-tools submodule now # for anonymize_parallel.sh -parallel +#parallel parquet-tools snappy From fb2903132ca2ff83d4e62cadc39012316c41c899 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:20:21 +0000 Subject: [PATCH 0402/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c8bfc3bab..fab97843d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c8bfc3bab2d1625eb10d5fa8a37c5e908bc6d8bc +Subproject commit fab97843df459a58170c70b2d848223c9a8af8a5 From ec3720c2d78c9797f602008a25de3294876e1b2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:24:56 +0000 Subject: [PATCH 0403/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6d460ffcb..7db4ac021 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6d460ffcb0d520ccf1b07d0e0a4762fff45da61d +Subproject commit 7db4ac0217fa34b38884d768fd3de38b9463117d From 9b30771321beb067c1705847f2207e2aab005741 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:30:51 +0000 Subject: [PATCH 0404/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fab97843d..2b0439d88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fab97843df459a58170c70b2d848223c9a8af8a5 +Subproject commit 2b0439d886154c961e51154a3e73ae4717493b13 From aa81f784bd9e8cc7502e634e898a2f551c00a5b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:30:51 +0000 Subject: [PATCH 0405/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7db4ac021..1f0b5a030 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7db4ac0217fa34b38884d768fd3de38b9463117d +Subproject commit 1f0b5a030216b9e08e63016ffeeb368256badb79 From 8c505b12f44abb2a73787055decaa361f87ae749 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 12:08:37 +0000 Subject: [PATCH 0406/2295] updated impala_tables_locations.py --- impala_tables_locations.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 1182164c6..63bae3e0f 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -19,6 +19,13 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first hundred or so tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 2820ce8d9e331b6b55a15aac4a01930131e85df7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 12:08:52 +0000 Subject: [PATCH 0407/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 8182f74e1..792be37d4 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,6 +19,13 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first hundred or so tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 1089f2645a6e1db24ce20f858ece60c09ef0adc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 13:53:15 +0000 Subject: [PATCH 0408/2295] updated impala_tables_locations.py --- impala_tables_locations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 63bae3e0f..12c0a93f2 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -23,7 +23,7 @@ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't - Impala is faster than Hive for the first hundred or so tables but then slows down + Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From d470418d29d4e66253dcbf8aa9e7f62a71b1c58c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 13:53:20 +0000 Subject: [PATCH 0409/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 792be37d4..2baa580e8 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -23,7 +23,7 @@ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't - Impala is faster than Hive for the first hundred or so tables but then slows down + Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From aba0a3aa0a01c9680dfbb142da16e2a7268269df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 14:03:40 +0000 Subject: [PATCH 0410/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index ccf66fc2a..04b89e728 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveTablesMetadata(HiveForEachTable): @@ -94,7 +94,7 @@ def process_options(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): log.info("describing table '%s.%s'", database, table) - location = 'UNKNOWN' + field = 'UNKNOWN' with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) @@ -103,9 +103,9 @@ def execute(self, conn, database, table, query): table_cursor.execute(query.format(table=table)) for row in table_cursor: if self.field.search(row[0]): - location = row[1] + field = row[1] break - print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + print('{db}.{table}\t{field}'.format(db=database, table=table, field=field)) if __name__ == '__main__': From df8aee7355e3082df710c4e4aafec6fcaea99f0e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:20:56 +0000 Subject: [PATCH 0411/2295] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9185ade80..c701d780c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) +[![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) +[![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) From 23632588d56824170c41f7853357b5b11ff41ddf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0412/2295] added ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_14.04.yaml diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml new file mode 100644 index 000000000..c03f7072e --- /dev/null +++ b/.github/workflows/ubuntu_14.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 14.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:14.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 56be868bee97a34d0f4454f3229fa8e197f764b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0413/2295] added ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_16.04.yaml diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml new file mode 100644 index 000000000..714dbbaf7 --- /dev/null +++ b/.github/workflows/ubuntu_16.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 16.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:16.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From fcffe68c0b1b4732925e2cadd9d395c81790ca96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0414/2295] added ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_18.04.yaml diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml new file mode 100644 index 000000000..d3ab2752c --- /dev/null +++ b/.github/workflows/ubuntu_18.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 18.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:18.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From b11e7dbd5049a56fccfa5ff5eef786a6675d1324 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 14:31:58 +0000 Subject: [PATCH 0415/2295] added auto-determining repo from local git remotes --- travis_last_log.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/travis_last_log.py b/travis_last_log.py index b9b6843f2..5005c3417 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -61,6 +61,7 @@ #import string import sys import traceback +import git srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) @@ -76,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class TravisLastBuildLog(CLI): @@ -147,7 +148,9 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') validate_alnum(self.travis_token, 'travis token') self.headers['Authorization'] = 'token {0}'.format(self.travis_token) self.num = self.get_opt('num') @@ -163,6 +166,18 @@ def process_options(self): #if not self.color and not (sys.__stdin__.isatty() and sys.__stdout__.isatty()): # self.plaintext = True + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if self.job_id: self.print_log(job_id=self.job_id) From 0a2d66ca6df74608a7a57e6adb7144412a24800d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 14:37:47 +0000 Subject: [PATCH 0416/2295] added auto-determining repo from local git remotes --- travis_debug_session.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index 6812eeead..eb2557d7d 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -48,6 +48,7 @@ import sys import time import traceback +import git try: import requests except ImportError: @@ -68,7 +69,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.9.0' class TravisDebugSession(CLI): @@ -151,10 +152,24 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') validate_alnum(self.travis_token, 'travis token') self.headers['Authorization'] = 'token {0}'.format(self.travis_token) + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if not self.job_id: if self.repo: From a659c16927d2520747e23e41035cc52b9a3f756c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 15:12:00 +0000 Subject: [PATCH 0417/2295] added .github/workflows/fedora.yaml --- .github/workflows/fedora.yaml | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/fedora.yaml diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml new file mode 100644 index 000000000..8ae839658 --- /dev/null +++ b/.github/workflows/fedora.yaml @@ -0,0 +1,55 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Fedora + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: fedora + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 18f9be55279662e8c25e2ecaa6d005df5fac78f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 15:19:24 +0000 Subject: [PATCH 0418/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c701d780c..cf203535d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Hari Sekhon - DevOps Python Tools [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) +[![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From b867dd3b94a55ca32c170c867d68e0cafa37e420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:48 +0000 Subject: [PATCH 0419/2295] sync'd workflows from bash-tools --- .github/workflows/alpine.yaml | 8 +++++--- .github/workflows/centos.yaml | 8 +++++--- .github/workflows/centos6.yaml | 10 +++++++--- .github/workflows/centos7.yaml | 8 +++++--- .github/workflows/centos8.yaml | 8 +++++--- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 9b188de5a..7988da349 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: alpine + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index daf5d8d57..8d3c00564 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index 5856a69d4..f2e27b9d4 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -11,6 +11,8 @@ # https://www.linkedin.com/in/harisekhon # +# Not supporting RHEL6 any more because it doesn't have GNU parallel package + name: CI CentOS 6 #env: @@ -30,6 +32,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:6 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +46,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index f92b232d4..3939b04f2 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:7 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 2016b60d6..407ead97f 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:8 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test From 6d1fd6655d5fe66e60264698c60db17a91dc79b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:51 +0000 Subject: [PATCH 0420/2295] added alpine_3.yaml --- .github/workflows/alpine_3.yaml | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/alpine_3.yaml diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml new file mode 100644 index 000000000..da8a373a3 --- /dev/null +++ b/.github/workflows/alpine_3.yaml @@ -0,0 +1,55 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Alpine 3 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: alpine:3 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apk add --no-cache git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From a3e2da25d098c40c31e22f40e2004c70ab1afe15 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:51 +0000 Subject: [PATCH 0421/2295] added debian.yaml --- .github/workflows/debian.yaml | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian.yaml diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml new file mode 100644 index 000000000..43392f0c5 --- /dev/null +++ b/.github/workflows/debian.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 37dfcaefeefe51f53c539bcfefa133efaa621dec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:52 +0000 Subject: [PATCH 0422/2295] added debian_10.yaml --- .github/workflows/debian_10.yaml | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_10.yaml diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml new file mode 100644 index 000000000..31775a833 --- /dev/null +++ b/.github/workflows/debian_10.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 10 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:10-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 2a458bfb8f129de6c50103e519ff39e435986265 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:52 +0000 Subject: [PATCH 0423/2295] added debian_6.yaml --- .github/workflows/debian_6.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_6.yaml diff --git a/.github/workflows/debian_6.yaml b/.github/workflows/debian_6.yaml new file mode 100644 index 000000000..f0ec8860c --- /dev/null +++ b/.github/workflows/debian_6.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 6 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:6 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + echo VERSION: ; cat /etc/*release /etc/*version 2>/dev/null || : + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 8a4c6438a9264cf4a5951ea76f9b38370580a995 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:53 +0000 Subject: [PATCH 0424/2295] added debian_7.yaml --- .github/workflows/debian_7.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_7.yaml diff --git a/.github/workflows/debian_7.yaml b/.github/workflows/debian_7.yaml new file mode 100644 index 000000000..640d28a79 --- /dev/null +++ b/.github/workflows/debian_7.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 7 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:7-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 674f9be36ced36c905906c5ef11dbb37423dd290 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:53 +0000 Subject: [PATCH 0425/2295] added debian_8.yaml --- .github/workflows/debian_8.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_8.yaml diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml new file mode 100644 index 000000000..b191b7355 --- /dev/null +++ b/.github/workflows/debian_8.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 8 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:8-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 6fab28d1465bfbd549b565d706d424c81e51641c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:54 +0000 Subject: [PATCH 0426/2295] added debian_9.yaml --- .github/workflows/debian_9.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_9.yaml diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml new file mode 100644 index 000000000..3308dec1b --- /dev/null +++ b/.github/workflows/debian_9.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 9 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:9-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From d64ea52a1c810cf006869a8577fd062bca2ffb97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:21:05 +0000 Subject: [PATCH 0427/2295] updated README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf203535d..5e6ed34a9 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,23 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) -[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) +[![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) +[![CI Debian 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+6%22) +[![CI Debian 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+7%22) +[![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) +[![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) +[![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) +[![CI Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine+3%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From 1a7f1af1ff18abd92b62ceb1971d130393334495 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 14:11:58 +0000 Subject: [PATCH 0428/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 2baa580e8..54d79dbe9 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -25,6 +25,8 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs ~1.5x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From 262835ba24253c3341aee0a2edef1bc033684e45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:00 +0000 Subject: [PATCH 0429/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2b0439d88..cee0464e9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2b0439d886154c961e51154a3e73ae4717493b13 +Subproject commit cee0464e95f806f77b70207de51972d3f6780052 From a96c839704b4374c30500cb836d295b15a487fab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:00 +0000 Subject: [PATCH 0430/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1f0b5a030..e4434cfc6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1f0b5a030216b9e08e63016ffeeb368256badb79 +Subproject commit e4434cfc6be2e5a9b17252d0a2c0d0f98b5f3574 From ed161ee726a2a65654be01a485a677b07be621d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:08 +0000 Subject: [PATCH 0431/2295] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5e6ed34a9..54e939f7a 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,6 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) -[![CI Debian 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+6%22) -[![CI Debian 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+7%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) [![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) From 58783f76db75b085c2170b99eb96c98255fb480c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:12 +0000 Subject: [PATCH 0432/2295] renamed debian_6.yaml to debian_6.yamldisabled --- .github/workflows/{debian_6.yaml => debian_6.yamldisabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_6.yaml => debian_6.yamldisabled} (100%) diff --git a/.github/workflows/debian_6.yaml b/.github/workflows/debian_6.yamldisabled similarity index 100% rename from .github/workflows/debian_6.yaml rename to .github/workflows/debian_6.yamldisabled From b82102310bba71d9e4ff06398577850cdb2c572b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:12 +0000 Subject: [PATCH 0433/2295] renamed debian_7.yaml to debian_7.yamldisabled --- .github/workflows/{debian_7.yaml => debian_7.yamldisabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_7.yaml => debian_7.yamldisabled} (100%) diff --git a/.github/workflows/debian_7.yaml b/.github/workflows/debian_7.yamldisabled similarity index 100% rename from .github/workflows/debian_7.yaml rename to .github/workflows/debian_7.yamldisabled From a73ffa05c22013884029ec5312f207bad1c1b7d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:33:17 +0000 Subject: [PATCH 0434/2295] renamed debian_6.yamldisabled to debian_6.yaml.disabled --- .../workflows/{debian_6.yamldisabled => debian_6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_6.yamldisabled => debian_6.yaml.disabled} (100%) diff --git a/.github/workflows/debian_6.yamldisabled b/.github/workflows/debian_6.yaml.disabled similarity index 100% rename from .github/workflows/debian_6.yamldisabled rename to .github/workflows/debian_6.yaml.disabled From c01cafbdc50b195ce378283176e8a2788315164f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:33:17 +0000 Subject: [PATCH 0435/2295] renamed debian_7.yamldisabled to debian_7.yaml.disabled --- .../workflows/{debian_7.yamldisabled => debian_7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_7.yamldisabled => debian_7.yaml.disabled} (100%) diff --git a/.github/workflows/debian_7.yamldisabled b/.github/workflows/debian_7.yaml.disabled similarity index 100% rename from .github/workflows/debian_7.yamldisabled rename to .github/workflows/debian_7.yaml.disabled From 00c9c1ebd160fd17bf2a37f33044dcc8805cfd7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:34:37 +0000 Subject: [PATCH 0436/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e4434cfc6..ff66e8a3b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e4434cfc6be2e5a9b17252d0a2c0d0f98b5f3574 +Subproject commit ff66e8a3b243e798b507d1717c69927129e9daa1 From ff623990c621f3538442644a2bce73d2324d6c4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 15:07:24 +0000 Subject: [PATCH 0437/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cee0464e9..46e4a2ace 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cee0464e95f806f77b70207de51972d3f6780052 +Subproject commit 46e4a2acef5c34fc17cdab9350192cb8ee5f6e18 From ba7fb678acc165a4c1d6cbf73af13c826a95a91f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 15:07:24 +0000 Subject: [PATCH 0438/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ff66e8a3b..b5a9c16a2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ff66e8a3b243e798b507d1717c69927129e9daa1 +Subproject commit b5a9c16a2e05e5d6a645a7ccfd68de7ba1780955 From 4ae8df003113c3e6c27c91d888ece149a54bd5be Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 16:08:18 +0000 Subject: [PATCH 0439/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef9f2ce1b..887df140f 100755 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From ddb2c860c63d8c570a0238fe29efe1ef7a7b1df6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 17:49:27 +0000 Subject: [PATCH 0440/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5b419ce79..824b319a4 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -30,6 +30,7 @@ jobs: timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, ubuntu-16.04] steps: From 905fe194874a67843dd31d7e7e00b4a06c8b690b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:38:28 +0000 Subject: [PATCH 0441/2295] updated mac.yaml --- .github/workflows/mac.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index db5e0d0da..7d67cd384 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,6 +37,8 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - name: brew update + run: which brew && brew update || : - name: build run: make - name: test From 66daa1e1949991b703c919216c8adb5f844cceaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:40:23 +0000 Subject: [PATCH 0442/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 824b319a4..fae40bf65 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,11 +28,7 @@ jobs: build: #name: build timeout-minutes: 10 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, ubuntu-16.04] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/cache@v1 From 5fe291961b89527e3ed312afc6d47ab73508ab52 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:51:12 +0000 Subject: [PATCH 0443/2295] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 7d67cd384..70cf40da8 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: macos-latest steps: - uses: actions/checkout@v2 From b898f46e00c3a5962642e52926de9bca8e9b6a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 10:49:43 +0000 Subject: [PATCH 0444/2295] updated workflows --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos6.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 4 ++-- .github/workflows/debian_8.yaml | 4 ++-- .github/workflows/debian_9.yaml | 4 ++-- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 6 +++--- .github/workflows/ubuntu.yaml | 10 +++++----- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- 16 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 7988da349..eee5180b8 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: alpine env: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index da8a373a3..45ba0ee60 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: alpine:3 env: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8d3c00564..b4bedd79d 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos env: diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index f2e27b9d4..eb053b667 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -29,7 +29,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:6 env: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 3939b04f2..2b3567790 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:7 env: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 407ead97f..807ea4d85 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:8 env: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 43392f0c5..c0d32c539 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: debian env: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 31775a833..24499f3ec 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:10-slim + container: debian:10 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index b191b7355..e2cad44dd 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:8-slim + container: debian:8 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 3308dec1b..f2d84ffb8 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:9-slim + container: debian:9 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 8ae839658..735a91d72 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: fedora env: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 70cf40da8..3632e946c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: 2020-02-04 21:39:47 +0000 (Tue, 04 Feb 2020) +# Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # @@ -34,9 +34,9 @@ jobs: - uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- + ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || : - name: build diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index fae40bf65..840e5553a 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # @@ -16,7 +16,7 @@ name: CI Ubuntu #env: # DEBUG: 1 -on: +on: # [push] push: branches: - master @@ -27,16 +27,16 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- + ${{ runner.os }}-pip-devops-python-tools - name: build run: make - name: test diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index c03f7072e..1ae4728b7 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:14.04 env: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 714dbbaf7..b7c311aca 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:16.04 env: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d3ab2752c..6de120ffa 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:18.04 env: From 3dc035d9a4abea14373d08fd1fc16d674de0a635 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 10:49:45 +0000 Subject: [PATCH 0445/2295] added mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/mac_10.15.yaml diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml new file mode 100644 index 000000000..e0b628e6a --- /dev/null +++ b/.github/workflows/mac_10.15.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Mac 10.15 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: brew update + run: which brew && brew update || : + - name: build + run: make + - name: test + run: make test From b3422c46b7e612e33c0081da221856e8a18df875 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 11:30:06 +0000 Subject: [PATCH 0446/2295] renamed centos6.yaml to centos6.yaml.disabled --- .github/workflows/{centos6.yaml => centos6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos6.yaml => centos6.yaml.disabled} (100%) diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml.disabled similarity index 100% rename from .github/workflows/centos6.yaml rename to .github/workflows/centos6.yaml.disabled From b2d8552d966e133eee50463b272ee90f30635b56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 11:30:15 +0000 Subject: [PATCH 0447/2295] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 54e939f7a..410e2f97f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Hari Sekhon - DevOps Python Tools [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) [![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) -[![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) From 58b244411de1721e89036a651b92b417bb997083 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 13:52:07 +0000 Subject: [PATCH 0448/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 54d79dbe9..cb52cc8ed 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -26,7 +26,7 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead eg. by ~1900 tables the Hive version will overtake the Impala version and - for thousands of tables Impala actuallys runs ~1.5x slower than the Hive version overall + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From 465c2fba6c9dc819f3ee0f31446445b4b10c7994 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 13:52:17 +0000 Subject: [PATCH 0449/2295] updated impala_tables_locations.py --- impala_tables_locations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 12c0a93f2..3be05d6f8 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -25,6 +25,8 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From b349db6009ef6e7784df696443a115c1be40af8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 17:24:07 +0000 Subject: [PATCH 0450/2295] updated test_welcome.sh --- tests/test_welcome.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 9deb7ad8a..37e3d59af 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -24,6 +24,9 @@ cd "$srcdir/.."; # Fedora doesn't have /var/log/wtmp if ! [ -f /var/log/wtmp ]; then + echo "/var/log/wtmp doesn't exist, touching..." + # assigned in utils.sh + # shellcheck disable=SC2154 $sudo touch /var/log/wtmp || : fi From 4aa023e5e599118ccc3bfa7d4fe0415974b4ab60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 21:04:52 +0000 Subject: [PATCH 0451/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index e0b628e6a..a16b0731f 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,7 +38,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip-devops-python-tools - name: brew update - run: which brew && brew update || : + run: which brew && brew update || echo - name: build run: make - name: test From 6da2cd7b05d4aac47fc5becc6c752598f1756c9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 21:05:26 +0000 Subject: [PATCH 0452/2295] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 3632e946c..0c1358164 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,7 +38,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip-devops-python-tools - name: brew update - run: which brew && brew update || : + run: which brew && brew update || echo - name: build run: make - name: test From 6f87ad2da49c8200569c7021dbb6c12293a56182 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 10:30:32 +0000 Subject: [PATCH 0453/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46e4a2ace..a85df1ca3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46e4a2acef5c34fc17cdab9350192cb8ee5f6e18 +Subproject commit a85df1ca3a6202c409707a8d9e45e4cc6d2c8b3f From 1e8e410c0ab371777b01301ee9f723456e284d16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 10:30:32 +0000 Subject: [PATCH 0454/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b5a9c16a2..3100736a8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b5a9c16a2e05e5d6a645a7ccfd68de7ba1780955 +Subproject commit 3100736a87133e99e4beea098574c8280f610a0d From 6c5992b08311adc4383261d304bccf79fbf591ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 14:15:41 +0000 Subject: [PATCH 0455/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a85df1ca3..feddf4f12 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a85df1ca3a6202c409707a8d9e45e4cc6d2c8b3f +Subproject commit feddf4f12d344257fb37c8eb548bdad736c11512 From 68331800b8882c2f6d2e23c3fdd029d34edb4da5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 14:15:41 +0000 Subject: [PATCH 0456/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3100736a8..85117d262 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3100736a87133e99e4beea098574c8280f610a0d +Subproject commit 85117d2628bf56105d984c82c3aca8437140e12a From c02354c9a1a6ae2b48258916bc6551e7a19a1b4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:00 +0000 Subject: [PATCH 0457/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index cb52cc8ed..c81b9305d 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,6 +19,11 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Examples: + +./impala_tables_metadata.py --field Location ... +./impala_tables_metadata.py --field SerDe ... + Caveats: Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't From 0ea0eb2205fbdde8aef52d4cef45b68451679662 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:06 +0000 Subject: [PATCH 0458/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 04b89e728..a77972b16 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,6 +19,11 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes +Examples: + +./hive_tables_metadata.py --field Location ... +./hive_tables_metadata.py --field SerDe ... + Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 5113ad70582888c697109774a9cc6e5a49cebe8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:20 +0000 Subject: [PATCH 0459/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index c81b9305d..659d0244a 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,7 +19,7 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes -Examples: +Examples (fields are case sensitive): ./impala_tables_metadata.py --field Location ... ./impala_tables_metadata.py --field SerDe ... From 6dec5d3e4bae527c4136f0bb5096c196707820cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:23 +0000 Subject: [PATCH 0460/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index a77972b16..4755f829e 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,7 +19,7 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes -Examples: +Examples (fields are case sensitive): ./hive_tables_metadata.py --field Location ... ./hive_tables_metadata.py --field SerDe ... From 9f7973532b45a5ed4b44f03a1e814cd74bc55ecc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:25:47 +0000 Subject: [PATCH 0461/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 4755f829e..36afc9875 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,7 +19,7 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes -Examples (fields are case sensitive): +Examples (fields are case sensitive regex and return N/A without match): ./hive_tables_metadata.py --field Location ... ./hive_tables_metadata.py --field SerDe ... @@ -99,7 +99,7 @@ def process_options(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): log.info("describing table '%s.%s'", database, table) - field = 'UNKNOWN' + field = 'N/A' with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) From f70ef2d79210de1227b83c5b704db8d388cb33f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:25:52 +0000 Subject: [PATCH 0462/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 659d0244a..09f79c484 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,7 +19,7 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes -Examples (fields are case sensitive): +Examples (fields are case sensitive regex and return N/A without match): ./impala_tables_metadata.py --field Location ... ./impala_tables_metadata.py --field SerDe ... From 0c9ebd8520311a9de819ff1f5d5ad6908c9523ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:29:46 +0000 Subject: [PATCH 0463/2295] updated impala_tables_metadata.py --- impala_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 09f79c484..ba8a66cbc 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -16,8 +16,8 @@ """ -Connect to an Impala daemon and list the locations of all tables in all databases, -or only those matching given db / table regexes +Connect to an Impala daemon and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes Examples (fields are case sensitive regex and return N/A without match): From a12f51855eeeaf45c098cbe16e77459ed1198422 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:29:52 +0000 Subject: [PATCH 0464/2295] updated hive_tables_metadata.py --- hive_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 36afc9875..886c3eed8 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -16,8 +16,8 @@ """ -Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') -for all tables in all databases, or only those matching given db / table regexes +Connect to HiveServer2 and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes Examples (fields are case sensitive regex and return N/A without match): From b678761256f268d0178a554cd9141ad3b5f7f914 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:30:33 +0000 Subject: [PATCH 0465/2295] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 7b35b0174..a47d846cc 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -16,7 +16,7 @@ """ -Connect to an Impala daemon and execute a query for all tables in all databases, +Connect to an Impala daemon and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From a81722383dffea5eba41872d1efa5051e29dc811 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:30:38 +0000 Subject: [PATCH 0466/2295] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 4ef224290..1b40f4434 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Connect to HiveServer2 and execute a query for all tables in all databases, +Connect to HiveServer2 and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From c642b3c6dcca537ddc6eb9d6982547703596e77d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:05:08 +0000 Subject: [PATCH 0467/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index feddf4f12..c07860544 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit feddf4f12d344257fb37c8eb548bdad736c11512 +Subproject commit c07860544f350a9fc1e63fbc799baf1bb8957459 From 8c9164729302e76c9ae382d1feac1e095f41427a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:05:08 +0000 Subject: [PATCH 0468/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85117d262..34f4467de 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85117d2628bf56105d984c82c3aca8437140e12a +Subproject commit 34f4467ded90be3770e9cb447de7cb715e640a8e From 6bf152fbee61b7d7730cee9516a2fa3cf3a7a09f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:25:53 +0000 Subject: [PATCH 0469/2295] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index db112cbc7..c956b3189 100644 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -25,3 +25,5 @@ python2-psutil #python-flask #python-markupsafe #python2-bitarray + +python3-snappy From 10977a0645988311fc7327cfeba1689c66fadadf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 10:00:21 +0000 Subject: [PATCH 0470/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c07860544..40a3466bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c07860544f350a9fc1e63fbc799baf1bb8957459 +Subproject commit 40a3466bbf7ad0c34fbb467f8f25757265ddf166 From b2632a3ab2020716399c6d47f4ad8e92b891d350 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 10:00:22 +0000 Subject: [PATCH 0471/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 34f4467de..d4df4ea2b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 34f4467ded90be3770e9cb447de7cb715e640a8e +Subproject commit d4df4ea2b8a9160ce990d1737eeeefaea378ed19 From 77a635e32db2ecdecf7627944f094b127f6e43c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:04:19 +0000 Subject: [PATCH 0472/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d4df4ea2b..432f81312 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d4df4ea2b8a9160ce990d1737eeeefaea378ed19 +Subproject commit 432f813127a62295abc9e1b25b7b535288656dd6 From cc1c4bd3a594b637274e81a2d75c8f32dbfd7a34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:50:54 +0000 Subject: [PATCH 0473/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 40a3466bb..14c1c4033 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 40a3466bbf7ad0c34fbb467f8f25757265ddf166 +Subproject commit 14c1c40335c2f1f86c4e43bfbd5b45c0b530d322 From 855d19342fc6bfade64d809ddc2f64a6bd8a228a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:50:55 +0000 Subject: [PATCH 0474/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 432f81312..6faf9c1c5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 432f813127a62295abc9e1b25b7b535288656dd6 +Subproject commit 6faf9c1c5b24ee26a782d4066b83656f8c3ff4ae From 26358ae940f758c9486759c7ed74b92ec9bacb24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:02:04 +0000 Subject: [PATCH 0475/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14c1c4033..69316f67e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14c1c40335c2f1f86c4e43bfbd5b45c0b530d322 +Subproject commit 69316f67e12349ec9f4d704758d9edd313d33a2a From 79d659e5b7ae03a2521cd980885af524d6295de5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:02:05 +0000 Subject: [PATCH 0476/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6faf9c1c5..76913aba1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6faf9c1c5b24ee26a782d4066b83656f8c3ff4ae +Subproject commit 76913aba121992e1bb59dfb7adf1888c361e2fa3 From fc69f4f2419fce5421b1f66b6130284e864dff69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:57:47 +0000 Subject: [PATCH 0477/2295] added hive_tables_column_counts.py --- hive_tables_column_counts.py | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100755 hive_tables_column_counts.py diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py new file mode 100755 index 000000000..0f993e60a --- /dev/null +++ b/hive_tables_column_counts.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count the number of columns for each table in each database, +or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + column_cursor.execute('describe `{}`'.format(table)) + for _ in column_cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + print('{db}.{table}\t{column_count}'.format(db=database, table=table, column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesColumnCounts().main() From f29c560c04e7fa442a431bd805e29cc3e9689b74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:57:50 +0000 Subject: [PATCH 0478/2295] added impala_tables_column_counts.py --- impala_tables_column_counts.py | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 impala_tables_column_counts.py diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py new file mode 100755 index 000000000..5f9e6d486 --- /dev/null +++ b/impala_tables_column_counts.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Connect to an Impala daemon and count the number of columns for each table in each database, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_column_counts import HiveTablesColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesColumnCounts(HiveTablesColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesColumnCounts().main() From 7d93a76ea7e3954e6afaeb8638d8b5ac28aa0877 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:59:04 +0000 Subject: [PATCH 0479/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 410e2f97f..21d3c5383 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations + - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs From 40f72c104b78fd5e308163e246f8b0490bfd39d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:01:22 +0000 Subject: [PATCH 0480/2295] updated hive_tables_column_counts.py --- hive_tables_column_counts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 0f993e60a..100984804 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -60,6 +60,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesColumnCounts(HiveForEachTable): def __init__(self): From 295ef72c050a8cf07cab50de27299165cec08303 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:01:29 +0000 Subject: [PATCH 0481/2295] updated hive_tables_list.py --- hive_tables_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_list.py b/hive_tables_list.py index b0c4f4702..07c415780 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -63,6 +63,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.1.0' + class HiveTablesList(HiveForEachTable): def __init__(self): From 7858272398f0d859f6f6e99bfdca198acadc0b4b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:06 +0000 Subject: [PATCH 0482/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index a78e049f4..84c0cc92b 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -67,6 +67,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesNullColumns(HiveForEachTable): def __init__(self): From 6b99408da172da05f24f16770a0f4c22c9e3357d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:16 +0000 Subject: [PATCH 0483/2295] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 78c42d15e..8368e3c02 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -64,6 +64,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesNullRows(HiveForEachTable): def __init__(self): From 0c3c130b6b405a040efd6e2df671b97b81eb91d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:32 +0000 Subject: [PATCH 0484/2295] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 0f7a5a94a..7266d0b1e 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -64,6 +64,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesRowsWithNulls(HiveForEachTable): def __init__(self): From 3638bf19c6ba10869e75bc3619f006755dee1077 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:54 +0000 Subject: [PATCH 0485/2295] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 47c104789..3d748ded2 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -62,6 +62,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class ImpalaTablesRowCounts(HiveTablesRowCounts): def __init__(self): From 8d5f177c00b4d18bbd9ac8411f6a0b62f8a9144e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:08:41 +0000 Subject: [PATCH 0486/2295] updated impala_tables_column_counts.py --- impala_tables_column_counts.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index 5f9e6d486..2489e4e9c 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -16,9 +16,6 @@ """ -Connect to an Impala daemon and find tables with columns containing only NULLs -for all tables in all databases, or only those matching given db / table regexes - Connect to an Impala daemon and count the number of columns for each table in each database, or only those matching given db / table regexes From ddce6027a12431a7738920d1c55c0384a033146b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:10:21 +0000 Subject: [PATCH 0487/2295] added hive_tables_row_column_counts.py --- hive_tables_row_column_counts.py | 110 +++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 hive_tables_row_column_counts.py diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py new file mode 100755 index 000000000..660fef904 --- /dev/null +++ b/hive_tables_row_column_counts.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Output format: + + .
+ +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesRowColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + row_count = None + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as cursor: + # doesn't support parameterized query quoting from dbapi spec + #cursor.execute('use %(database)s', {'database': database}) + #cursor.execute('describe %(table)s', {'table': table}) + cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + cursor.execute('describe `{}`'.format(table)) + for _ in cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table) + # doesn't support parameterized query quoting from dbapi spec + cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table)) + for result in cursor: + assert row_count is None + row_count = result[0] + print('{db}.{table}\t{row_count}\t{column_count}'\ + .format(db=database, + table=table, + row_count=row_count, + column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesRowColumnCounts().main() From 5e865f04e6e606580bdbc5cc515e375da18c0753 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:10:25 +0000 Subject: [PATCH 0488/2295] added impala_tables_row_column_counts.py --- impala_tables_row_column_counts.py | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_row_column_counts.py diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py new file mode 100755 index 000000000..b2e867734 --- /dev/null +++ b/impala_tables_row_column_counts.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_column_counts import HiveTablesRowColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesRowColumnCounts(HiveTablesRowColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowColumnCounts().main() From 794415305e756fb9b79f5fa23b84e143bb83cf84 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:16:50 +0000 Subject: [PATCH 0489/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 21d3c5383..9b159e02a 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables + - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs From 9d111fa28667faca5c5dfd5f4906e9da399001ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 16:40:40 +0000 Subject: [PATCH 0490/2295] fix for Python 3 behaviour change around string => bytes --- tests/test_anonymize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 8d67fcf36..6850e650b 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -40,8 +40,12 @@ def run(): print('running anonymize tests using: {} {}'.format(anonymize, args)) cmd = [anonymize] + [_ for _ in args.split()] process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) + # encode as bytes for Python 3 :-/ + test_input = str.encode(test_input, 'utf-8') (stdout, _) = process.communicate(input=test_input) index = 0 + # convert bytes to string + stdout = stdout.decode("utf-8") for line in stdout.split('\n'): # pylint: disable=redefined-outer-name key = src_keys[index] _input = src[key] From 41f852c4d600a2551e9e2515ba757ae812fbb768 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:44:21 +0000 Subject: [PATCH 0491/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 69316f67e..37e99e112 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 69316f67e12349ec9f4d704758d9edd313d33a2a +Subproject commit 37e99e1120f420820972b312154b3b0b4fd0817c From 5149cfcecc8d0def68613ecae0616334a6748089 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:44:22 +0000 Subject: [PATCH 0492/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76913aba1..dc632d05f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76913aba121992e1bb59dfb7adf1888c361e2fa3 +Subproject commit dc632d05fe529d90b8e41de55ddfb0fe46267991 From 4b1497593f23f42fc8aacb82b4975ff223ac172f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:48:30 +0000 Subject: [PATCH 0493/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dc632d05f..e1862f618 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dc632d05fe529d90b8e41de55ddfb0fe46267991 +Subproject commit e1862f618101114f92c668ef8263bf1a4ce0e2db From ea71e676c92f62ebc7be49a643733e4f83f8ac56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:20:32 +0000 Subject: [PATCH 0494/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37e99e112..35e2e9a7a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37e99e1120f420820972b312154b3b0b4fd0817c +Subproject commit 35e2e9a7a5b90528fcebcfcd76c2f1c4c777c8d1 From 5ca5fb46a1258e1c9324d4a4cded63d5ef0de17d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:20:32 +0000 Subject: [PATCH 0495/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e1862f618..46e0dcf96 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e1862f618101114f92c668ef8263bf1a4ce0e2db +Subproject commit 46e0dcf961daae7351bcf5a570ad09aa132fa6b9 From 791a132e3327ff269399676df9a9cfa6e34c3b8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:30:05 +0000 Subject: [PATCH 0496/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 46e0dcf96..16f89c774 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 46e0dcf961daae7351bcf5a570ad09aa132fa6b9 +Subproject commit 16f89c77475f05ce2cf24468d14b5f2b31f4e2cf From cb8b2a3ed4f20165d6c5e1324047371ba39577f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:32:58 +0000 Subject: [PATCH 0497/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 887df140f..2b4b72a4e 100755 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi bash-tools/python_pip_install.sh "avro-python3" || : @# for impyla From ea0b3e9e446f179c6b34930885112b30c9cda3a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:46:28 +0000 Subject: [PATCH 0498/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 35e2e9a7a..fa52c2bef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 35e2e9a7a5b90528fcebcfcd76c2f1c4c777c8d1 +Subproject commit fa52c2bef66dc0966a46a300f7a2325398850eba From e094d084550ee6155bdacaae8299c22150127e01 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:46:28 +0000 Subject: [PATCH 0499/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 16f89c774..e409fa0b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 16f89c77475f05ce2cf24468d14b5f2b31f4e2cf +Subproject commit e409fa0b5a8434b534e341d98227ca3a8930004d From 8f9068bb16099adbe08c1a61523c4060a80c74a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:52:13 +0000 Subject: [PATCH 0500/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e409fa0b5..532ae5f8f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e409fa0b5a8434b534e341d98227ca3a8930004d +Subproject commit 532ae5f8fd36dd7c9e54d432bc537d271cc9002c From 725002ebcc5db351fa353675adbce25b1e4dc546 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 13:46:01 +0000 Subject: [PATCH 0501/2295] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77613f9cb..696aa5de9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,8 @@ python-snappy==0.5 sasl==0.2.1 sh==1.12.14 # pulls in python-KrbV as a dependency which doesn't build on Mac any more -# moved to Makefile as best effort +# relies on python-krbV is unmaintained and unported to Python 3 +# - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 snakebite==2.11.0 thrift-sasl==0.2.1 From 96bc66bd1a5697dbc5d6778bb86fe5febe5a0eea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 13:46:34 +0000 Subject: [PATCH 0502/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b4b72a4e..f6bb403d4 100755 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ python: @#$(SUDO_PIP) pip install -r requirements.txt @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt - @# python-krbV dependency doesn't build on Mac any more and is unmaintained + @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit @bash-tools/python_pip_install.sh snakebite[kerberos] || : From ed5b34473b21e675133659968e9b50e9a3af14b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 15:00:13 +0000 Subject: [PATCH 0503/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa52c2bef..35fccb2dc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa52c2bef66dc0966a46a300f7a2325398850eba +Subproject commit 35fccb2dce79b470373adc20da840fd6bccc7386 From 70957696d220ddb689b2c108eda52ca7e91e45f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 15:00:13 +0000 Subject: [PATCH 0504/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 532ae5f8f..d091e31a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 532ae5f8fd36dd7c9e54d432bc537d271cc9002c +Subproject commit d091e31a557ac9c5da729e7ae21fe2cd98a31ab6 From 7c146709ce3199bf53c308f629faefb072f5313f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 Feb 2020 14:52:32 +0000 Subject: [PATCH 0505/2295] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index 88bcf4c3a..0ab2bc769 100644 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -16,6 +16,7 @@ py-dicttoxml py-psutil py-pyldap -py2-jinja2 -py2-numpy -#py-flask +py3-jinja2 +py3-numpy +#py3-flask +py3-pygit2 From 7eb28b2b0430a5d41f143e7c84ca62f1c4ee085b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 13:15:48 +0000 Subject: [PATCH 0506/2295] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9b159e02a..bcc685a6f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ Hari Sekhon - DevOps Python Tools [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 046e0798e7e8fc4addc6e253b109624b4a8bed53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 14:20:55 +0000 Subject: [PATCH 0507/2295] added bootstrap.sh --- setup/bootstrap.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 setup/bootstrap.sh diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh new file mode 100755 index 000000000..c8b387861 --- /dev/null +++ b/setup/bootstrap.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# Alpine / Wget: +# +# wget https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh && sh bootstrap.sh +# +# Curl: +# +# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "$0")" + +repo="https://github.com/HariSekhon/DevOps-Python-tools" + +directory="pytools" + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type apk >/dev/null 2>&1; then + apk --no-cache add bash git make + elif type apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y git make + elif type yum >/dev/null 2>&1; then + yum install -y git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +if [ "${srcdir##*/}" = setup ]; then + cd "$srcdir/.." +elif [ -d "$directory" ]; then + cd pytools +else + git clone "$repo" "$directory" + cd "$directory" +fi + +make From d0fece87532923e62de1d2ad6182743fab159daa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 14:25:16 +0000 Subject: [PATCH 0508/2295] updated Makefile --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f6bb403d4..afe57b99c 100755 --- a/Makefile +++ b/Makefile @@ -15,17 +15,21 @@ # =================== # bootstrap commands: +# setup/bootstrap.sh +# +# OR +# # Alpine: # -# apk add --no-cache git $(MAKE) && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# # Debian / Ubuntu: # -# apt-get update && apt-get install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# # RHEL / CentOS: # -# yum install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) +# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make # =================== From 05fe5cdb21386fc8b279cd9c0d149d9bc23d5230 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 15:21:03 +0000 Subject: [PATCH 0509/2295] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 696aa5de9..28d238a6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ dicttoxml==1.7.4 #elasticsearch>=1.0.0,<2.0.0 # fails on requiring newer version of setuptools #Flask==0.10.1 -GitPython==2.1.14 +GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 From d0270aae2cdafbcf142a7d5092501e82d16171a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 15:45:11 +0000 Subject: [PATCH 0510/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bcc685a6f..b638ec10b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) -[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 04417ce4292cb50a475e3d03091909f807733fd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:02:29 +0000 Subject: [PATCH 0511/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 35fccb2dc..aded5fdee 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 35fccb2dce79b470373adc20da840fd6bccc7386 +Subproject commit aded5fdeea7705f90aa3e5b5f1f96ad65f144d4b From 47af104ca60efcee561e6f10efb7511b9fbce8c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:02:30 +0000 Subject: [PATCH 0512/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d091e31a5..432e7d1a0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d091e31a557ac9c5da729e7ae21fe2cd98a31ab6 +Subproject commit 432e7d1a0e348601829edda48119a02a0778d76c From e61c50b6a67df1e66f16e391a9223b61a1d8e795 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:13:33 +0000 Subject: [PATCH 0513/2295] updated impala_tables_column_counts.py --- impala_tables_column_counts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index 2489e4e9c..b26b0ebcc 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -19,6 +19,10 @@ Connect to an Impala daemon and count the number of columns for each table in each database, or only those matching given db / table regexes +You can also get this from the schemas.csv output generated by impala_schemas_csv.py, eg. + + tail -n +2 impala_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 58ebf363d27ea68511f9948f09209b6bb919aff7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:13:56 +0000 Subject: [PATCH 0514/2295] updated hive_tables_column_counts.py --- hive_tables_column_counts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 100984804..2902f23dd 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -19,6 +19,10 @@ Connect to HiveServer2 and count the number of columns for each table in each database, or only those matching given db / table regexes +You can also get this from the schemas.csv output generated by hive_schemas_csv.py, eg. + + tail -n +2 hive_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From c4739bda418e7632b764c0229e36cb3667b94e0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 13:02:36 +0000 Subject: [PATCH 0515/2295] updated dockerhub_search.py --- dockerhub_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerhub_search.py b/dockerhub_search.py index cb226e59f..405981dab 100755 --- a/dockerhub_search.py +++ b/dockerhub_search.py @@ -56,7 +56,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6' +__version__ = '0.6.1' class DockerHubSearch(CLI): @@ -71,7 +71,7 @@ def __init__(self): self.quiet = False def add_options(self): - self.add_opt('-n', '--num', '--limit', default=50, + self.add_opt('-n', '--num', '--limit', default=50, type=int, help='Number of results to return (default: 50)') self.add_opt('-q', '--quiet', action='store_true', help='Output only the image names, one per line (useful for shell scripting)') From ec0d785cb5641fe9918dcabd1e76ffed67654f46 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 18:01:08 +0000 Subject: [PATCH 0516/2295] updated bootstrap.sh --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index c8b387861..a77c27e19 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -53,7 +53,7 @@ fi if [ "${srcdir##*/}" = setup ]; then cd "$srcdir/.." elif [ -d "$directory" ]; then - cd pytools + cd "$directory" else git clone "$repo" "$directory" cd "$directory" From 6286d1a796c2e17f8a08653ef8188418684d6395 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:12:00 +0000 Subject: [PATCH 0517/2295] added azure-pipelines.yml --- azure-pipelines.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..0517a2142 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,25 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: 'ubuntu-18.04' + +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" + displayName: docker build From 60a1f34cd658acbbf15e1839f2918bf31dc119c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:15:39 +0000 Subject: [PATCH 0518/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0517a2142..5c1e82316 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,5 +21,6 @@ trigger: pool: vmImage: 'ubuntu-18.04' +steps: - script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" displayName: docker build From a363ea6cc01c214d4e52df0362aef80f4ef17f2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:42:23 +0000 Subject: [PATCH 0519/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index aded5fdee..16025a4be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit aded5fdeea7705f90aa3e5b5f1f96ad65f144d4b +Subproject commit 16025a4bea20a14e7c03c0a836adec81c0e5969d From 4a6e52396f7935336b9552a5cc664d294dadd6d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:42:24 +0000 Subject: [PATCH 0520/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 432e7d1a0..573575e48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 432e7d1a0e348601829edda48119a02a0778d76c +Subproject commit 573575e4893a4116e13fe74b92395376d5305f4c From ce957375c7c3dd1d8228486e6c598028c3efbd89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:53:09 +0000 Subject: [PATCH 0521/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5c1e82316..f0ecbeb5c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,5 +22,5 @@ pool: vmImage: 'ubuntu-18.04' steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make default test" displayName: docker build From 845a016d403991a260e63a3a39784f14f23629d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 18:01:47 +0000 Subject: [PATCH 0522/2295] added .appveyor.yml --- .appveyor.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..d3790490b --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,28 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://www.appveyor.com/docs/appveyor-yml/ + +image: Ubuntu + +install: +- sudo apt update +- sudo apt install -y git make +- make + +test_script: +- make test + +build: off From fee24cae6c2dc30e9dd87b683671e96695587701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:21:28 +0000 Subject: [PATCH 0523/2295] added codefresh.yml --- codefresh.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 codefresh.yml diff --git a/codefresh.yml b/codefresh.yml new file mode 100644 index 000000000..ceff141da --- /dev/null +++ b/codefresh.yml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +version: "1.0" +stages: + - "checkout" + - "build" +steps: + checkout: + type: "git-clone" + description: "Cloning main repository..." + repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' + revision: "${{CF_REVISION}}" + stage: "checkout" + build: + title: Running docker image + type: freestyle + working_directory: '${{CF_REPO_NAME}}' + arguments: + image: 'ubuntu:18.04' + commands: + - apt update && apt install -y git make + - make + - make test From 5fbb7698009acf2ad3711dae831f30c8f7ab3259 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:25:50 +0000 Subject: [PATCH 0524/2295] added shippable.yml --- shippable.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml new file mode 100644 index 000000000..215035109 --- /dev/null +++ b/shippable.yml @@ -0,0 +1,35 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# http://docs.shippable.com/platform/workflow/config/ + +language: none + +branches: + only: + - master + +build: + ci: + - shippable_retry make + - make test + +integrations: + notifications: + - integrationName: email + type: email + on_success: never + on_failure: never + on_pull_request: never From e46b39b8e5c31b1ec1ddd9f8a9f7529d614eb977 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:30:24 +0000 Subject: [PATCH 0525/2295] added wercker.yml --- wercker.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml new file mode 100644 index 000000000..9c7eb0961 --- /dev/null +++ b/wercker.yml @@ -0,0 +1,30 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://devcenter.wercker.com/reference/wercker-yml/ + +box: debian + +build: + steps: + - script: + name: install git & make + code: apt-get update && apt-get install -y git make + - script: + name: build + code: make + - script: + name: test + code: make test From f38054379caa53345bde90db3c305cf0d197906e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:41:16 +0000 Subject: [PATCH 0526/2295] added config.yml --- .circleci/config.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..41b107303 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,28 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020) +# Original: H1 2016 (Circle CI 1.x) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://circleci.com/docs/2.0/configuration-reference + +version: 2.1 +jobs: + build: + machine: + #image: ubuntu-1604:201903-01 + image: default + steps: + - checkout + - run: make + - run: make test From 879d29ad3d4986fea5e0b5079d9d15b0ebe0fad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:11 +0000 Subject: [PATCH 0527/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f0ecbeb5c..1967d5718 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,5 +22,5 @@ pool: vmImage: 'ubuntu-18.04' steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make default test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make ci test" displayName: docker build From 77e5c37f12b5b73e2adcc05cefbb0f19f57bc51d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:12 +0000 Subject: [PATCH 0528/2295] updated codefresh.yml --- codefresh.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codefresh.yml b/codefresh.yml index ceff141da..35dd12f02 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -32,5 +32,6 @@ steps: image: 'ubuntu:18.04' commands: - apt update && apt install -y git make - - make + - make init + - make ci - make test From 61c84eb91646386eccc35f9238514becae3fb4d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:12 +0000 Subject: [PATCH 0529/2295] updated shippable.yml --- shippable.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 215035109..68a3c8ef1 100644 --- a/shippable.yml +++ b/shippable.yml @@ -23,7 +23,9 @@ branches: build: ci: - - shippable_retry make + #- shippable_retry make + - make init + - make ci - make test integrations: From 573204fff51c4d9ecb0ee3edad6ddedfdbaaccac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:13 +0000 Subject: [PATCH 0530/2295] updated wercker.yml --- wercker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 9c7eb0961..c1037b720 100644 --- a/wercker.yml +++ b/wercker.yml @@ -22,9 +22,12 @@ build: - script: name: install git & make code: apt-get update && apt-get install -y git make + - script: + name: init + code: make init - script: name: build - code: make + code: make ci - script: name: test code: make test From 7b4833a04908a1c2ee382bcc53f4ba57f4144021 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:13 +0000 Subject: [PATCH 0531/2295] updated .circleci/config.yml --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41b107303..78c5f45d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/bash-tools # # License: see accompanying Hari Sekhon LICENSE file # @@ -24,5 +24,6 @@ jobs: image: default steps: - checkout - - run: make + - run: make init + - run: make ci - run: make test From 06a9bc84f58fc709c17383bb6645eab080a519ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:38:51 +0000 Subject: [PATCH 0532/2295] added bitbucket-pipelines.yml --- bitbucket-pipelines.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 000000000..2fd727146 --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,26 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://confluence.atlassian.com/x/5Q4SMw +# Only use spaces to indent your .yml configuration. +# ----- +# You can specify a custom docker image from Docker Hub as your build environment. +image: atlassian/default-image:2 + +pipelines: + default: + - step: + script: + - apt update && apt install -y git make && make init && make ci test From 169393bf618628271ca4e0cafc18274c6f1673c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:57:01 +0000 Subject: [PATCH 0533/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 16025a4be..c2bea1feb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 16025a4bea20a14e7c03c0a836adec81c0e5969d +Subproject commit c2bea1feb574904153069cbddaf9ef87cbcfaa24 From db5443f31647b81e8221e5f12e12d4ba9c200228 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:57:02 +0000 Subject: [PATCH 0534/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 573575e48..2518c2c3f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 573575e4893a4116e13fe74b92395376d5305f4c +Subproject commit 2518c2c3f551c8063c6af3f2d9836a250728d4d2 From 8c8ed18c635896d4f9f0c1d848876d7ed1f7b25e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:58:08 +0000 Subject: [PATCH 0535/2295] Initial Bitbucket Pipelines configuration From dafbd7115e48934996274ec9bdbbb580f7517569 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 12:07:04 +0000 Subject: [PATCH 0536/2295] added .sonarcloud.properties --- .sonarcloud.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 .sonarcloud.properties diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..a4496869e --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.host.url=https://sonarcloud.io From 443ad32cf1b2e33373c85466f205ae950b4c395f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 14:27:06 +0000 Subject: [PATCH 0537/2295] added .drone.yml --- .drone.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..4dd6036f6 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +kind: pipeline +type: docker +name: default + +steps: +- name: build + image: ubuntu:18.04 +# environment: +# DEBUG: 1 + commands: + - apt update -qq + - apt install -qy git make + - make init + - make ci + - make test + +trigger: + branch: + - master From 3a6d75a42e6ae4fe130f53b6ccb05c61617162f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:28:30 +0000 Subject: [PATCH 0538/2295] updated README.md --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b638ec10b..3a5f95e27 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Build Status](https://travis-ci.org/HariSekhon/DevOps-Python-tools.svg?branch=master)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) @@ -14,6 +13,19 @@ Hari Sekhon - DevOps Python Tools [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) +[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) +[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) +[![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) +[![Shippable](https://api.shippable.com/projects/5e52c63445c70f0007ff5144/badge?branch=master)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) + +[![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/DevOps-Python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) + [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) From c8076a4579104485271dbc71bde8a4d30ec6ab4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:02 +0000 Subject: [PATCH 0539/2295] sync'd CI builds --- .github/workflows/alpine.yaml | 20 ++++++-------------- .github/workflows/alpine_3.yaml | 20 ++++++-------------- .github/workflows/centos.yaml | 20 ++++++-------------- .github/workflows/centos6.yaml.disabled | 20 ++++++-------------- .github/workflows/centos7.yaml | 20 ++++++-------------- .github/workflows/centos8.yaml | 20 ++++++-------------- .github/workflows/debian.yaml | 21 ++++++--------------- .github/workflows/debian_10.yaml | 21 ++++++--------------- .github/workflows/debian_8.yaml | 21 ++++++--------------- .github/workflows/debian_9.yaml | 21 ++++++--------------- .github/workflows/fedora.yaml | 20 ++++++-------------- .github/workflows/ubuntu_14.04.yaml | 21 ++++++--------------- .github/workflows/ubuntu_16.04.yaml | 21 ++++++--------------- .github/workflows/ubuntu_18.04.yaml | 21 ++++++--------------- azure-pipelines.yml | 18 +++++++++++++++++- bitbucket-pipelines.yml | 2 +- codefresh.yml | 4 +++- shippable.yml | 2 ++ wercker.yml | 2 +- 19 files changed, 108 insertions(+), 207 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index eee5180b8..b4ea64cf0 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apk add --no-cache git make + run: apk add --no-cache git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 45ba0ee60..b9d1305a4 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apk add --no-cache git make + run: apk add --no-cache git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index b4bedd79d..13ff19170 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled index eb053b667..6ec523536 100644 --- a/.github/workflows/centos6.yaml.disabled +++ b/.github/workflows/centos6.yaml.disabled @@ -38,20 +38,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 2b3567790..7ebef0745 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 807ea4d85..3868ce406 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index c0d32c539..9969fb0a1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 24499f3ec..7980bae04 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index e2cad44dd..a7acf16cc 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index f2d84ffb8..e03e57161 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 735a91d72..1f7100849 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 1ae4728b7..f04e0bfda 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index b7c311aca..c122a9cab 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 6de120ffa..ff17f01cd 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1967d5718..cdf98c65a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,8 +19,24 @@ trigger: - master pool: + # there is no /dev/stderr on this azure build! + #vmImage: 'ubuntu-latest' + # Ubuntu 16.04 required for docker container support, looks like 18.04 works too vmImage: 'ubuntu-18.04' +# unprivileged container without sudo, cannot install dependencies +#container: ubuntu:18.04 + +# doesn't work due to lack of sudo +#steps: +#- script: sudo apt-get update && sudo apt-get install -y git make +# displayName: install git & make +#- script: make +# displayName: build +#- script: make test +# displayName: test + +# hacky workaround to Azure Pipelines limitations :-( steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make ci test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update -qq && apt install -qy git make && make init && make ci test" displayName: docker build diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 2fd727146..f44aee45d 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -23,4 +23,4 @@ pipelines: default: - step: script: - - apt update && apt install -y git make && make init && make ci test + - apt update -qq && apt install -qy git make && make init && make ci test diff --git a/codefresh.yml b/codefresh.yml index 35dd12f02..70990129d 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,8 @@ # https://www.linkedin.com/in/harisekhon # +# https://codefresh.io/docs/docs/codefresh-yaml/ + version: "1.0" stages: - "checkout" @@ -31,7 +33,7 @@ steps: arguments: image: 'ubuntu:18.04' commands: - - apt update && apt install -y git make + - apt update -qq && apt install -qy git make - make init - make ci - make test diff --git a/shippable.yml b/shippable.yml index 68a3c8ef1..6bcd8a238 100644 --- a/shippable.yml +++ b/shippable.yml @@ -15,6 +15,8 @@ # http://docs.shippable.com/platform/workflow/config/ +# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ + language: none branches: diff --git a/wercker.yml b/wercker.yml index c1037b720..18b14ed8f 100644 --- a/wercker.yml +++ b/wercker.yml @@ -21,7 +21,7 @@ build: steps: - script: name: install git & make - code: apt-get update && apt-get install -y git make + code: apt-get update -qq && apt-get install -qy git make - script: name: init code: make init From 65aafde9667cbbbada8d45234e49a23684befc71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:18 +0000 Subject: [PATCH 0540/2295] sync'd CI builds --- .appveyor.yml | 4 ++-- .circleci/config.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d3790490b..e367a94c7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,8 +18,8 @@ image: Ubuntu install: -- sudo apt update -- sudo apt install -y git make +- sudo apt update -qq +- sudo apt install -qy git make - make test_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index 78c5f45d8..9c36ac51b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/bash-tools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From 410c54c0a7e6e7114be0d5d8c6c2860b7edd9004 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:35 +0000 Subject: [PATCH 0541/2295] added .gitlab-ci.yml --- .gitlab-ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..1613d1410 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# https://docs.gitlab.com/ee/ci/yaml/README.html + +image: ubuntu:18.04 + +job: + before_script: + - apt-get update -qq && apt-get install -yq git make + script: + - make init && make ci test From 3914bf32cf3901a8f1cfcd782b85f0720d776293 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:36 +0000 Subject: [PATCH 0542/2295] added .cirrus.yml --- .cirrus.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..3006fac0c --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,20 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +container: + image: ubuntu:18.04 + +task: + script: sudo apt update -qq && apt install -qy git make && make init && make ci test From fb08cdc53e2bda1d22405f60adfa4ce77e7bb346 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:43:42 +0000 Subject: [PATCH 0543/2295] updated . --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c36ac51b..7217d95a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,5 +25,5 @@ jobs: steps: - checkout - run: make init - - run: make ci + - run: make - run: make test From 29b85f7e14f7fce80ee24f5b02eaf907f266245b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:44:00 +0000 Subject: [PATCH 0544/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a5f95e27..2a5f68d6c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/DevOps-Python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 462a073630d74b8f889f17cf038673d46627316b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:45:29 +0000 Subject: [PATCH 0545/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a5f68d6c..4c566f3e6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 1cb7107f72059cd0cca4cd0fe131e1a9e184071e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 18:29:21 +0000 Subject: [PATCH 0546/2295] fix for Python 3 --- crunch_accounting_csv_statement_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 477198dbf..9755b4435 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.3' +__version__ = '0.7.0' class CrunchAccountingCsvStatementConverter(CLI): @@ -157,7 +157,7 @@ def reverse_contents(filename): return tmp_filename def detect_columns(self, csvreader): - headers = csvreader.next() + headers = next(csvreader) if headers[0][0] == '{': log.error('JSON opening braces detected, not a CSV?') return False @@ -224,7 +224,7 @@ def validate_csvreader(csvreader, filename): log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 - except csv.Error as _: + except csv.Error as _: log.warning('file %s, line %s: %s', filename, csvreader.line_num, _) return None if count == 0: From 8acf4d0f3955a7d3cac4d6f7903220f08c54ef4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 17:45:05 +0000 Subject: [PATCH 0547/2295] added urlencode.py --- urlencode.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 urlencode.py diff --git a/urlencode.py b/urlencode.py new file mode 100755 index 000000000..c24b0b5c9 --- /dev/null +++ b/urlencode.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to url encode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import quote +else: + from urllib import quote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class URLEncode(CLI): + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.encode(arg) + else: + for line in sys.stdin: + line = line.rstrip('\n').rstrip('\r') + self.encode(line) + + @staticmethod + def encode(string): + #print(urllib.parse.quote(string)) + print(quote(string)) + +if __name__ == '__main__': + URLEncode().main() From 1f5bc87bced8cc7ee96aeda4045a47f3f9f259da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 17:54:10 +0000 Subject: [PATCH 0548/2295] updated urlencode.py --- urlencode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urlencode.py b/urlencode.py index c24b0b5c9..0b7987367 100755 --- a/urlencode.py +++ b/urlencode.py @@ -41,9 +41,9 @@ # pylint: disable=no-name-in-module,import-error if isPythonMinVersion(3): - from urllib.parse import quote + from urllib.parse import quote_plus as quote else: - from urllib import quote + from urllib import quote_plus as quote __author__ = 'Hari Sekhon' __version__ = '0.1.0' From 7c50b20f2db0a9b640095687f0d2e1c903c5d34d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 18:04:55 +0000 Subject: [PATCH 0549/2295] updated urlencode.py --- urlencode.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/urlencode.py b/urlencode.py index 0b7987367..ea8ca96ae 100755 --- a/urlencode.py +++ b/urlencode.py @@ -48,6 +48,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.1.0' + class URLEncode(CLI): def run(self): @@ -64,5 +65,6 @@ def encode(string): #print(urllib.parse.quote(string)) print(quote(string)) + if __name__ == '__main__': URLEncode().main() From 831c413ae98db1cde53bb049d0a260a8403df481 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 18:09:36 +0000 Subject: [PATCH 0550/2295] added urldecode.py --- urldecode.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 urldecode.py diff --git a/urldecode.py b/urldecode.py new file mode 100755 index 000000000..d2b9d89a3 --- /dev/null +++ b/urldecode.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to url decode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import unquote_plus as unquote +else: + from urllib import unquote_plus as unquote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class URLDecode(CLI): + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.decode(arg) + else: + for line in sys.stdin: + line = line.rstrip('\n').rstrip('\r') + self.decode(line) + + @staticmethod + def decode(string): + print(unquote(string)) + + +if __name__ == '__main__': + URLDecode().main() From 974fe6a639d8262f7275bf86f7bbdf08cfb73ef8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:39:15 +0000 Subject: [PATCH 0551/2295] updated center.py --- center.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 752db7f6a..8ac287184 100755 --- a/center.py +++ b/center.py @@ -42,7 +42,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.4.1' class Center(CLI): @@ -53,6 +53,7 @@ def __init__(self): # super().__init__() self.re_bound = re.compile(r'(\b)') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.timeout_default = None def add_options(self): self.add_opt('-w', '--width', default=80, type='int', metavar='', From ceedd3626c57274b6c0295521b09fec4522b1451 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:42:38 +0000 Subject: [PATCH 0552/2295] updated urlencode.py --- urlencode.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/urlencode.py b/urlencode.py index ea8ca96ae..756dee829 100755 --- a/urlencode.py +++ b/urlencode.py @@ -46,11 +46,18 @@ from urllib import quote_plus as quote __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class URLEncode(CLI): + def __init__(self): + # Python 2.x + super(URLEncode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + def run(self): if len(sys.argv) > 1: for arg in sys.argv[1:]: @@ -64,6 +71,7 @@ def run(self): def encode(string): #print(urllib.parse.quote(string)) print(quote(string)) + sys.stdout.flush() if __name__ == '__main__': From fc0c2c9ac46e835e7af69ea131c78e02127c628d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:42:51 +0000 Subject: [PATCH 0553/2295] updated urldecode.py --- urldecode.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/urldecode.py b/urldecode.py index d2b9d89a3..0f9d1e0f0 100755 --- a/urldecode.py +++ b/urldecode.py @@ -46,11 +46,18 @@ from urllib import unquote_plus as unquote __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class URLDecode(CLI): + def __init__(self): + # Python 2.x + super(URLDecode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + def run(self): if len(sys.argv) > 1: for arg in sys.argv[1:]: @@ -63,6 +70,7 @@ def run(self): @staticmethod def decode(string): print(unquote(string)) + sys.stdout.flush() if __name__ == '__main__': From 4c57c347923f4e9244c12da4fe3c819f4332c5f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:46:29 +0000 Subject: [PATCH 0554/2295] worked around stdin buffering to avoid Control-D char clashing --- urldecode.py | 7 ++++++- urlencode.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/urldecode.py b/urldecode.py index 0f9d1e0f0..4dd2242b7 100755 --- a/urldecode.py +++ b/urldecode.py @@ -63,7 +63,12 @@ def run(self): for arg in sys.argv[1:]: self.decode(arg) else: - for line in sys.stdin: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break line = line.rstrip('\n').rstrip('\r') self.decode(line) diff --git a/urlencode.py b/urlencode.py index 756dee829..90d6ab5e6 100755 --- a/urlencode.py +++ b/urlencode.py @@ -63,7 +63,12 @@ def run(self): for arg in sys.argv[1:]: self.encode(arg) else: - for line in sys.stdin: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break line = line.rstrip('\n').rstrip('\r') self.encode(line) From f03972770bc52cc3297dad025aa17d0dd65e2176 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 10:38:47 +0000 Subject: [PATCH 0555/2295] updated .cirrus.yml --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 3006fac0c..38752ca97 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,4 +17,4 @@ container: image: ubuntu:18.04 task: - script: sudo apt update -qq && apt install -qy git make && make init && make ci test + script: apt update -qq && apt install -qy git make && make init && make ci test From adb9a8071164581b4e1674a4ea3269c25a4e8ffa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:00:23 +0000 Subject: [PATCH 0556/2295] updated test_apache-drill.sh --- tests/test_apache-drill.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index c9a675207..fd336dc2c 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -82,6 +82,6 @@ test_apache_drill(){ echo } -startupwait 70 +startupwait 120 run_test_versions "Apache Drill" From 623aded98ac969659180ffa84857b92b62a65a37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:05 +0000 Subject: [PATCH 0557/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c2bea1feb..6d0113404 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c2bea1feb574904153069cbddaf9ef87cbcfaa24 +Subproject commit 6d0113404f799d1309a0a4ff868e88c9b33adf0c From 8081871fe5dc7a75872074d8c18b54c7ee6b696d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:07 +0000 Subject: [PATCH 0558/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2518c2c3f..2f193da9e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2518c2c3f551c8063c6af3f2d9836a250728d4d2 +Subproject commit 2f193da9ea9c3aa2af8285bc560490009d5bb17c From 28eed4dd8ab0f2f45d8ca1f2ccb316c3e904492b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:35 +0000 Subject: [PATCH 0559/2295] updated excluded.sh --- tests/excluded.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/excluded.sh b/tests/excluded.sh index 68f62c58f..a85efb228 100755 --- a/tests/excluded.sh +++ b/tests/excluded.sh @@ -23,7 +23,9 @@ set -eu isExcluded(){ local prog="$1" [[ "$prog" =~ ^\* ]] && return 0 - [[ "$prog" =~ spark_.*.py ]] && return 0 + [[ "$prog" =~ spark_.*\.py ]] && return 0 + [[ "$prog" =~ \.jy ]] && return 0 + [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3 #[[ $prog =~ ipython-notebook ]] && return 0 # this external git check is expensive, skip it when in CI as using fresh git checkouts is_CI && return 1 From d992bcb82d44f65daea7dc59ae64d38df07131d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:04:12 +0000 Subject: [PATCH 0560/2295] updated help.sh --- tests/help.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/help.sh b/tests/help.sh index 7b4edd786..8666b576c 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -33,10 +33,8 @@ for x in ${@:-$(echo ./*.py 2>/dev/null)}; do echo; hr if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue - [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue - [[ "$x" =~ (hive|impala)_foreach_table.py$ ]] && continue - [[ "$x" =~ (hive|impala)_tables_row_counts.py$ ]] && continue [[ "$x" =~ pythonpath.py$ ]] && continue + [[ "$x" =~ aws_s3_presign.py$ ]] && continue elif [ $status = 1 ]; then if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && ! python -c 'import krbV'; then # best effort, not available on Mac any more From bc398ab2af6ed8ab33f14146002fc06e519ddcc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 17:12:25 +0000 Subject: [PATCH 0561/2295] updated welcome.py --- welcome.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/welcome.py b/welcome.py index 0d93b836f..bde562ac9 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.2' +__version__ = '2.0.3' class Welcome(CLI): @@ -121,7 +121,7 @@ def print_welcome(self): print(msg) return try: - charmap = list(string.uppercase + string.lowercase + '@#$%^&*()') + charmap = list(string.ascii_uppercase + string.ascii_lowercase + '@#$%^&*()') # print '', # print('', end='') for i in range(0, len(msg)): From 3b8abb83bd19d965f71997b444062a3f4d0ce05c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:09:50 +0000 Subject: [PATCH 0562/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d0113404..dd6b9b20b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d0113404f799d1309a0a4ff868e88c9b33adf0c +Subproject commit dd6b9b20b52bd5c0daf9b3fec57b3ffef4f67ad3 From 92d0a3df6109a23cbba61ffa2206e2860d010fb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:09:51 +0000 Subject: [PATCH 0563/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2f193da9e..4118be214 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2f193da9ea9c3aa2af8285bc560490009d5bb17c +Subproject commit 4118be214098593bc27e8411634dca5236b0e206 From 661e2d45d214b0915ee2d7e2b4c84ab7549614bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:10:17 +0000 Subject: [PATCH 0564/2295] updated all.sh --- tests/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index 7aa1f8c75..b1e1957dc 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -29,7 +29,8 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; -bash-tools/check_all.sh +# has to be included so that isExcluded function is inherited +. bash-tools/check_all.sh #tests/test_yamllint.sh From 194fe41d5d03f3838b9b341a2441812cd7777a32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:52:11 +0000 Subject: [PATCH 0565/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index dd6b9b20b..c447bad9b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit dd6b9b20b52bd5c0daf9b3fec57b3ffef4f67ad3 +Subproject commit c447bad9bd012b61a363647338abd95e2f2a8ab4 From c80ae2a12a3319f8087aacbb5d7dcaa9c42a6f67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:52:11 +0000 Subject: [PATCH 0566/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4118be214..f226a567a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4118be214098593bc27e8411634dca5236b0e206 +Subproject commit f226a567aa86ecb8ce73dab1b75b038ae7b6e347 From 73c1f321b54593df42d2f324c06fa448fb120ae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:55:54 +0000 Subject: [PATCH 0567/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c447bad9b..93c88e317 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c447bad9bd012b61a363647338abd95e2f2a8ab4 +Subproject commit 93c88e3177f22de869e02529390d7aff1b95dfb9 From 43af4dd3ef22f9adcd38eb533dae54646da65e14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:55:54 +0000 Subject: [PATCH 0568/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f226a567a..fcf075f55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f226a567aa86ecb8ce73dab1b75b038ae7b6e347 +Subproject commit fcf075f55e11856efb4982d3098e3eeada525310 From 5cb11a3f5290b22cd446074371af05f64de40d65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:31:03 +0000 Subject: [PATCH 0569/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 93c88e317..266052396 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 93c88e3177f22de869e02529390d7aff1b95dfb9 +Subproject commit 2660523969ad17ad8864973a4c390afe6ccd266b From 84db42ec1eac9b1b1f94e0c4b070b96dba3a5ecd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:31:05 +0000 Subject: [PATCH 0570/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fcf075f55..37044fa45 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fcf075f55e11856efb4982d3098e3eeada525310 +Subproject commit 37044fa45bcd4aa11090c2bdee06f6e40225b879 From fc639e6718f55b083cb9ae0ed380ec79c695ba0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:38:20 +0000 Subject: [PATCH 0571/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 266052396..43528a5b4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2660523969ad17ad8864973a4c390afe6ccd266b +Subproject commit 43528a5b4b1de2e2e2805ef0c5c6ebc203a0edd8 From 5dd80cd20d832cd9f349e315377504945afd46f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:38:39 +0000 Subject: [PATCH 0572/2295] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28d238a6e..299a1b6d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ sh==1.12.14 # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 -snakebite==2.11.0 +#snakebite==2.11.0 thrift-sasl==0.2.1 thrift==0.9.3 thriftpy==0.3.9 From 31317cddf9efffbe44050078a6f65b21171f5e55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:39:08 +0000 Subject: [PATCH 0573/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index afe57b99c..7d93ca278 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - @bash-tools/python_pip_install.sh snakebite[kerberos] || : + @bash-tools/setup/python_install_snakebite.sh # Python >= 3.4 - try but accept failure in case we're not on the right version of Python @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi From ed553cbe88238ba1e3d9a46074640437d37fdc02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:41:13 +0000 Subject: [PATCH 0574/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7d93ca278..c3689b47e 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - @bash-tools/setup/python_install_snakebite.sh + bash-tools/setup/python_install_snakebite.sh # Python >= 3.4 - try but accept failure in case we're not on the right version of Python @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi From 90f06ad0f247d8cf4f73ff64db5de5a3ea298a59 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 16:07:17 +0000 Subject: [PATCH 0575/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c566f3e6..819795fee 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,9 @@ Environment variables are supported for convenience and also to hide credentials - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```json_to_xml.py``` - JSON to XML converter - ```xml_to_json.py``` - XML to JSON converter + - ```json_to_xml.py``` - JSON to XML converter + - ```json_to_yaml.py``` - JSON to YAML converter - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - see also ```validate_*.py``` further down for all these formats and more - [Ambari](https://hortonworks.com/apache/ambari/): From f7b2f23022cf42be528791396bfeceebda778a73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 19:16:39 +0000 Subject: [PATCH 0576/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 43528a5b4..af8183da2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 43528a5b4b1de2e2e2805ef0c5c6ebc203a0edd8 +Subproject commit af8183da22d16bbbceef97764f7d03fd078b0475 From d1ee03ae8afffd3955ed7cc1ec8edb80a65b1195 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 19:16:39 +0000 Subject: [PATCH 0577/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 37044fa45..825ad2c93 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 37044fa45bcd4aa11090c2bdee06f6e40225b879 +Subproject commit 825ad2c93fc2733fab365557eb973d240b6678b7 From 264633e8b8583f22d8eaccc962c0b8454b6acf15 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 22:24:09 +0000 Subject: [PATCH 0578/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index af8183da2..f62b996e0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit af8183da22d16bbbceef97764f7d03fd078b0475 +Subproject commit f62b996e03be15d3907aa612da0c8577dbe914c8 From 303eca20af7866a669728705ff375649a3c2530f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 22:24:09 +0000 Subject: [PATCH 0579/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 825ad2c93..96b13e1c9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 825ad2c93fc2733fab365557eb973d240b6678b7 +Subproject commit 96b13e1c96789ce684cf865b57f9bd5324332bc0 From 52bfe12d65199431b13c35bd3af242593c2ebd9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:08:16 +0000 Subject: [PATCH 0580/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f62b996e0..34034522a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f62b996e03be15d3907aa612da0c8577dbe914c8 +Subproject commit 34034522a7e38cd7e41444fb21157a5f6c544625 From 2b6cf3bf30b270c7f5b1367b31ebd9e7575bace8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:08:16 +0000 Subject: [PATCH 0581/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 96b13e1c9..2b3bffa4b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 96b13e1c96789ce684cf865b57f9bd5324332bc0 +Subproject commit 2b3bffa4b91da25931deed6493f1915ac2aef861 From c5ccdf2c69d1010bba0b5f854c4f756f347fbf6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:41:37 +0000 Subject: [PATCH 0582/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34034522a..9cfc896b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34034522a7e38cd7e41444fb21157a5f6c544625 +Subproject commit 9cfc896b24982e442c2c7849c32c76589a56fc0c From 25afa9197cc7b7ec6fd28c307b30188adbc49cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:41:37 +0000 Subject: [PATCH 0583/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2b3bffa4b..73ec58d48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2b3bffa4b91da25931deed6493f1915ac2aef861 +Subproject commit 73ec58d48246f7c061227cf26dab9a6bb6a99601 From e8f03bd79da4b2c931c84ea52a143d5bc754cc1f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 09:35:20 +0000 Subject: [PATCH 0584/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9cfc896b2..690a0a4d2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9cfc896b24982e442c2c7849c32c76589a56fc0c +Subproject commit 690a0a4d2c25fca1f29bb87085e12d7c8b97aa5f From 4147702ca04d783e7760a90fc9b2d9f0b2002210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 09:35:20 +0000 Subject: [PATCH 0585/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 73ec58d48..5bf3bbc40 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 73ec58d48246f7c061227cf26dab9a6bb6a99601 +Subproject commit 5bf3bbc408df3f9e181043796547a83a6d383d32 From 5e0ae9dd6dcfc385fdd168ae780afd78cb859401 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:13:59 +0000 Subject: [PATCH 0586/2295] updated README.md --- README.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 819795fee..3b11a8439 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,42 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) + +[![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) + +[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) -[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) -[![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) -[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker)](https://hub.docker.com/r/harisekhon/github/) +[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) -[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) -[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) +[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://api.shippable.com/projects/5e52c63445c70f0007ff5144/badge?branch=master)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) +[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) From 5b5fb2e0c59c618404f4fbf5928e992adbdf51ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:17:58 +0000 Subject: [PATCH 0587/2295] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 3b11a8439..a1a1f11f7 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,12 @@ docker run harisekhon/pytools #### Automated Build from source ##### +installs git, make, pulls the repo and build the dependencies: +``` +curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh +``` + +or manually: ``` git clone https://github.com/harisekhon/devops-python-tools pytools cd pytools From 29c2d8ca5977ade7e96fe7be4d387d3b80a187a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:37:54 +0000 Subject: [PATCH 0588/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a1a1f11f7..bb5cf2ef2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Hari Sekhon - DevOps Python Tools ================================= [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 02dd9ec9569447c4f202e78bd5fac1ba9f586f3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:40:02 +0000 Subject: [PATCH 0589/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 690a0a4d2..d65b95ff3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 690a0a4d2c25fca1f29bb87085e12d7c8b97aa5f +Subproject commit d65b95ff3393906e7e7923787af56a89818a3f21 From 24263453aa8d6df9a4c185d3c72d2948f610ed8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:40:03 +0000 Subject: [PATCH 0590/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5bf3bbc40..b38575728 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5bf3bbc408df3f9e181043796547a83a6d383d32 +Subproject commit b385757288c78ce4a9024b353573f71ad7700fc3 From 6f9f291f039758cbd17f53f763f714feb1e896c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 22:56:05 +0000 Subject: [PATCH 0591/2295] updated aws_users_last_used.py --- aws_users_last_used.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 2657f03f7..588a1d7af 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -103,10 +103,10 @@ def run(self): break log.info('waiting for credentials report') time.sleep(1) - try: - result = iam.get_credential_report() - except ClientError as _: - raise + #try: + result = iam.get_credential_report() + #except ClientError as _: + # raise csv_content = result['Content'] log.debug('%s', csv_content) filehandle = StringIO(unicode(csv_content)) From fde38fda835c8451fdddd1721f48e133ca630b0c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 8 Mar 2020 22:56:18 +0000 Subject: [PATCH 0592/2295] Update psutil from 4.3.0 to 5.7.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 299a1b6d4..35cc9f30b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ ldif3==3.2.2 #numpy==1.17.2 numpy==1.16.5 python-cson==1.0.9 -psutil==4.3.0 +psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #PyHive==0.6.1 From cdadedd3479681a224e79464758e8318723b81cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:09 +0000 Subject: [PATCH 0593/2295] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 7266d0b1e..ce17ca99a 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -79,7 +79,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: From dfcf2d1195ce62ec6709b1a835c546b4b071753f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:36 +0000 Subject: [PATCH 0594/2295] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 8368e3c02..7ab8b53dd 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -79,7 +79,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: From 7546846094d89ea091016ea8a7bfecb2d1ae5d76 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:52 +0000 Subject: [PATCH 0595/2295] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 84c0cc92b..5874a1565 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -82,7 +82,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): sum_part = '' columns = [] log.info("describing table '%s.%s'", database, table) From 793c03f03d59ea06c01672b47b8b13ceb3b9c311 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 18:04:49 +0000 Subject: [PATCH 0596/2295] added cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 294 ++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 cloudera_navigator_tables_used.py diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py new file mode 100755 index 000000000..dc647a528 --- /dev/null +++ b/cloudera_navigator_tables_used.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Processes Cloudera Navigator exported CSV logs to list the tables used (selected from) + +This allows you to see if you wasting time maintaining datasets nobody is using + +Handles more than naive filtering delimited column numbers which will miss many table and database names: + + 1. table/database name fields are often blank and need to be inferred from SQL queries field + 2. SQL queries often contain newlines which break the rows up + 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used + +See cloudera_navigator_audit_download_logs.sh for a script to export these logs + +./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \ + navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv + +Output - CSV format to stdout: + +database,table + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +#import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import CriticalError, log + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ClouderaNavigatorTablesUsed(CLI): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsed, self).__init__() + # Python 3.x + # super().__init__() + self.delimiter = None + self.quotechar = None + self.escapechar = None + self.data = {} + self.timeout_default = None + + def add_options(self): + super(ClouderaNavigatorTablesUsed, self).add_options() + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' + # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour + self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use for outputting (default: ,)') + self.add_opt('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV output (recommended, default is double quote \'"\')') + self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)') + + def process_options(self): + super(ClouderaNavigatorTablesUsed, self).process_options() + self.delimiter = self.get_opt('delimiter') + self.quotechar = self.get_opt('quotechar') + self.escapechar = self.get_opt('escapechar') + if not self.args: + self.usage('no CSV file argument given') + + def run(self): + quoting = csv.QUOTE_ALL + if self.quotechar == '': + quoting = csv.QUOTE_NONE + + fieldnames = ['database', 'table'] + csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + for filename in self.args: + self.process_file(filename) + + csv_writer.writeheader() + for database in sorted(self.data): + for table in sorted(self.data[database]): + csv_writer.writerow({'database': database, + 'table': table}) + #if log.isEnabledFor(logging.DEBUG): + # sys.stdout.flush() + +# csv_header_indices.sh navigator_audit_2019_hive.csv +# 0 Timestamp +# 1 Username +# 2 "IP Address" +# 3 "Service Name" +# 4 Operation +# 5 Resource +# 6 Allowed +# 7 Impersonator +# 8 sub_operation +# 9 entity_id +# 10 stored_object_name +# 11 additional_info +# 12 collection_name +# 13 solr_version +# 14 operation_params +# 15 service +# 16 operation_text +# 17 url +# 18 operation_text +# 19 table_name +# 20 resource_path +# 21 database_name +# 22 object_type +# 23 Source +# 24 Destination +# 25 Permissions +# 26 "Delegation Token ID" +# 27 "Table Name" +# 28 Family +# 29 Qualifier +# 30 "Operation Text" +# 31 "Database Name" +# 32 "Table Name" +# 33 "Object Type" +# 34 "Resource Path" +# 35 "Usage Type" +# 36 "Operation Text" +# 37 "Query ID" +# 38 "Session ID" +# 39 Status +# 40 "Database Name" +# 41 "Table Name" +# 42 "Object Type" +# 43 Privilege + + # TODO: should really be refactored to be smaller simpler chunks of code + # XXX: this post processing is ugly as hell and probably brittle - YMMV + def process_file(self, filename): + re_select_from_table = re.compile(r'\bselect\b.+\bfrom\b(?:\s|\n)+([^\s,]+)', re.I | re.MULTILINE | re.DOTALL) + operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + ] + with open(filename) as csvfile: + csv_reader = csv.reader(csvfile, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert len_headers == 44 + operation_index = 4 + table_index = 19 + database_index = 21 + sql_index = 36 + assert headers[operation_index] == 'Operation' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'Operation Text' + partial_row = [] + sql_decomment = self.sql_decomment + # more complicated than I wish it was - msg me if you know a simpler cleaner way + for row in csv_reader: + #log.debug('row = %s', row) + #try: + # various logic to handle rows broken on newlines inside SQL queries + len_row = len(row) + if len_row > len_headers: + #log.debug('collapsing fields in row: %s', row) + difference = len_row - len_headers + row[sql_index] = ','.join([sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index + difference:] + len_row = len(row) + #log.debug('collapsed row: %s', row) + #log.debug('row length: %s', len_row) + #log.debug('partial row length: %s', len(partial_row)) + if len_row == len_headers: + pass + elif len_row < len_headers: + log.debug('row (partial): %s', row) + if len_row + len(partial_row) == len_headers + 1: + #log.debug('length row + partial_row == header length, completing partial row') + #log.debug('partial_row = %s', partial_row) + #log.debug('row = %s', row) + # join first field to last field to complete SQL query + sql_fragment = sql_decomment(row[0]) + partial_row[-1] = partial_row[-1] + r'\n ' + sql_fragment + partial_row += row[1:] + #log.debug('partial_row = %s', partial_row) + elif partial_row: + #log.debug('partial_row: %s', partial_row) + #log.debug('row: %s', row) + # join next fragment of SQL query to incomplete last item containing the first part of SQL query + partial_row[-1] = partial_row[-1] + r'\n ' + r'\n '.join(row) + #log.debug('accumulated partial row: %s', partial_row) + elif len(partial_row) > len_headers: + raise CriticalError('len(partial_row) > len_headers - {} > {} for partial row: {}'\ + .format(len(partial_row), len_headers, partial_row)) + else: + partial_row = row + #log.debug('partial_row = %s', partial_row) + #log.debug('len partial row = %s', len(partial_row)) + #log.debug('len headers = %s', len_headers) + if len(partial_row) == len_headers: + # process accumulated row as normal + row = partial_row + partial_row = [] + #log.debug('accumulated completed row: %s', row) + else: + continue + elif partial_row: + raise CriticalError('incompleted partial row: {}'.format(partial_row)) + len_row = len(row) + if len_row != len_headers: + raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + .format(len_row, len_headers, row)) + #log.debug(row) + database = row[21] + table = row[19] + if not table.strip(): + operation = row[4] + if operation == 'QUERY': + log.debug('table not found in row: %s', row) + query = row[36] + log.debug('trying to parse: %s', query) + match = re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + else: + log.warning('failed to parse table from query: %s', query) + elif operation in operations_to_ignore: + continue + else: + log.debug('table not found in row and operation is not a query to parse: %s', row) + if not table and not database: + continue + table = table.lower() + database = database.lower() + self.data[database] = self.data.get(database, {}) + self.data[database][table] = 1 + #except IndexError as _: +# if log.isEnabledFor(logging.DEBUG): +# log.error('%s - offending line: %s', _, row) +# else: + # raise CriticalError('ERROR: %s - offending line: %s', _, row) + + @staticmethod + def sql_decomment(string): + return string.split('--')[0] + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsed().main() From 6bdd602487bf01a75b8c290a51b7c151f2531a51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:52 +0000 Subject: [PATCH 0597/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb5cf2ef2..c580598ce 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ Hari Sekhon - DevOps Python Tools ================================= +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) From 0851091c709324e48ac749efb1975df65fdf788b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:59 +0000 Subject: [PATCH 0598/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d65b95ff3..40bb5733b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d65b95ff3393906e7e7923787af56a89818a3f21 +Subproject commit 40bb5733b5fef2ed71b86aac6be9a7235840f351 From 57848d6ed97ce0fc8e98532b8a6ed99bdb5dda13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:59 +0000 Subject: [PATCH 0599/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b38575728..f7ac21d1c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b385757288c78ce4a9024b353573f71ad7700fc3 +Subproject commit f7ac21d1c57d4e5b09b91007c9c9a0034e73fcb0 From 2c56c08c96db81be0b6be9065851227c42441f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:37 +0000 Subject: [PATCH 0600/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c580598ce..968e879fb 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ Hari Sekhon - DevOps Python Tools ================================= -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) From e1f599fb13bf0b3ca07458cc3c9533ffd604b735 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:45 +0000 Subject: [PATCH 0601/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 40bb5733b..cfbe5cbfd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 40bb5733b5fef2ed71b86aac6be9a7235840f351 +Subproject commit cfbe5cbfd7b4f2edf749416a7893ba34ce3b9d4e From 8da549398810c4a20f167c44ac2f01d4eef47fed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:46 +0000 Subject: [PATCH 0602/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f7ac21d1c..a2bc85bcd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f7ac21d1c57d4e5b09b91007c9c9a0034e73fcb0 +Subproject commit a2bc85bcdfc6dd1d34f4a76ba5d1a31775888dcd From 609aac048bb5e8fdc5873a1cdb74a76dd7d2ee03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:31 +0000 Subject: [PATCH 0603/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cfbe5cbfd..a9eadd361 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cfbe5cbfd7b4f2edf749416a7893ba34ce3b9d4e +Subproject commit a9eadd3614474214c613859d4c7c9c279afa8b72 From f07be1e873f597318e7a5f793696fc9e92cb8262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:32 +0000 Subject: [PATCH 0604/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a2bc85bcd..ca05a3266 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a2bc85bcdfc6dd1d34f4a76ba5d1a31775888dcd +Subproject commit ca05a32663d6ac37fc28a5d765c97f2c12bf4f10 From dc51ee693093811574a0eec243a8c9a91477a13a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:41 +0000 Subject: [PATCH 0605/2295] updated shippable.yml --- shippable.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/shippable.yml b/shippable.yml index 6bcd8a238..6b45ef91f 100644 --- a/shippable.yml +++ b/shippable.yml @@ -25,6 +25,13 @@ branches: build: ci: + # workaround to broken repos + # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB + # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 + # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. + # bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed + - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* + - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make - make init - make ci From f03b5c80a2b2b171c77a2d3c1430ac90dd767b08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:46 +0000 Subject: [PATCH 0606/2295] updated .appveyor.yml --- .appveyor.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index e367a94c7..d53f810d5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,6 +18,22 @@ image: Ubuntu install: +# workaround for: +# Some packages could not be installed. This may mean that you have +# requested an impossible situation or if you are using the unstable +# distribution that some required packages have not yet been created +# or been moved out of Incoming. +# The following information may help to resolve the situation: +# +# The following packages have unmet dependencies: +# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed +# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. +# bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# make[2]: *** [apt-packages] Error 123 +# make[2]: Leaving directory '/home/appveyor/projects/pylib' +# bash-tools/Makefile.in:212: recipe for target 'system-packages' failed +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list || : +- sudo apt purge -y mssql-server || : - sudo apt update -qq - sudo apt install -qy git make - make From f7711e36c93fa53d2b839acca420c89ea2399c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:47:03 +0000 Subject: [PATCH 0607/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a9eadd361..8a4e2e245 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a9eadd3614474214c613859d4c7c9c279afa8b72 +Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 From 876243882cd20b3d1a6fa8ae69be283db1760658 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:47:04 +0000 Subject: [PATCH 0608/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ca05a3266..b85f28276 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ca05a32663d6ac37fc28a5d765c97f2c12bf4f10 +Subproject commit b85f282766ad61a705984c06781e361154cb4cbf From 7e50bb844ed3d6c65843ed6f773a09ddbe5b438e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:48:35 +0000 Subject: [PATCH 0609/2295] updated .appveyor.yml .circleci/config.yml --- .appveyor.yml | 6 ++++-- .circleci/config.yml | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d53f810d5..c1e4e06ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,8 +32,10 @@ install: # make[2]: *** [apt-packages] Error 123 # make[2]: Leaving directory '/home/appveyor/projects/pylib' # bash-tools/Makefile.in:212: recipe for target 'system-packages' failed -- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list || : -- sudo apt purge -y mssql-server || : +# +# adding "|| :" to the end of these commands causes them to be silently ignored! +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list +- sudo apt purge -qy --allow-change-held-packages mssql-server - sudo apt update -qq - sudo apt install -qy git make - make diff --git a/.circleci/config.yml b/.circleci/config.yml index 7217d95a7..75c014403 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,9 +19,16 @@ version: 2.1 jobs: build: + # technically a docker image is a better choice than machine + # but we want to introduce some native environment variation + # between build systems in order to test the repo's build automation is robust machine: - #image: ubuntu-1604:201903-01 image: default + #image: ubuntu-1604:201903-01 + # set to an actual docker image when running locally using circle_ci_job.sh + # docker image must have git installed to do the checkout + # so using harisekhon/dev:ubuntu instead of base ubuntu image + #image: harisekhon/dev:ubuntu steps: - checkout - run: make init From 0a3c90d23de3d464ced58cb6fb160c0b5dc97a14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 14:22:24 +0000 Subject: [PATCH 0610/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 968e879fb..625d0aa42 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Hari Sekhon - DevOps Python Tools [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) From 43c4a71b92a4d5de3a68f29e2f32d40c727f96b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 18:08:41 +0000 Subject: [PATCH 0611/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8a4e2e245..e7ad87b8f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 +Subproject commit e7ad87b8f5d3c079014092a1eb468f6a7ef99cda From f11dd439e4490c2dc27b2a1f8be345a57411cba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 18:08:42 +0000 Subject: [PATCH 0612/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b85f28276..4805995eb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b85f282766ad61a705984c06781e361154cb4cbf +Subproject commit 4805995ebe22c6edde2a10553f8c18f074665a3d From 686217a7a82fceb607ff8b3e1450d7a598f2e4ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 20:15:20 +0000 Subject: [PATCH 0613/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8a4e2e245..2fda7589e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 +Subproject commit 2fda7589e4b4cf06b7b42467a1629f5e07812c5f From 66b376b2ecf16fbc26ab84c1131f695d3a82a36d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 20:15:20 +0000 Subject: [PATCH 0614/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b85f28276..46cab29f7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b85f282766ad61a705984c06781e361154cb4cbf +Subproject commit 46cab29f7354502cccd295589396a7394115b969 From 2a3a358d60e8ee6cebeeeba3055c6c8afc74a623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 23:13:28 +0000 Subject: [PATCH 0615/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2fda7589e..4a6277a0c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2fda7589e4b4cf06b7b42467a1629f5e07812c5f +Subproject commit 4a6277a0c21f817499d7181e0ad8e6f8f0a5699c From 70e5d480800809fcf0cca4e5086ff1b379bbdf13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 23:13:28 +0000 Subject: [PATCH 0616/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 46cab29f7..c1093e128 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 46cab29f7354502cccd295589396a7394115b969 +Subproject commit c1093e128c061e7d62f92f0455dcbb7c70af5564 From 6520be12e036ea58456b038475eeb429108b1462 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 13:20:16 +0000 Subject: [PATCH 0617/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 625d0aa42..da235c81a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) From 5e4986fc837d7ecedafbdc1462e4609ed67fe578 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:21:32 +0000 Subject: [PATCH 0618/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4a6277a0c..f72e25d7e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4a6277a0c21f817499d7181e0ad8e6f8f0a5699c +Subproject commit f72e25d7eb699c38d248525c36197f6d99bb96d2 From e0afc03b5c6dcd29530e5bf2f812216efd2b442f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:21:33 +0000 Subject: [PATCH 0619/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c1093e128..7de1ce1dd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c1093e128c061e7d62f92f0455dcbb7c70af5564 +Subproject commit 7de1ce1ddd165b8f91b285df5cce9a49fbba7b42 From 2556e5b9c72b4dc50285ca554b783306ec31e6e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:41:14 +0000 Subject: [PATCH 0620/2295] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 35cc9f30b..5df77ddb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ ldif3==3.2.2 # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 +psycopg2==2.8.4 python-cson==1.0.9 psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From aef4fbedcae40b67e6f4eed2a61e15ce308b705b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:58:35 +0000 Subject: [PATCH 0621/2295] added lib/postgres_cli.py --- lib/postgres_cli.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 lib/postgres_cli.py diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py new file mode 100755 index 000000000..68864934f --- /dev/null +++ b/lib/postgres_cli.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import socket +import sys +import psycopg2 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class PostgreSQLCLI(CLI): + + def __init__(self): + # Python 2.x + super(PostgreSQLCLI, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['PostgreSQL', 'Postgres', 'PG'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 5432 + self.user = None + self.password = None + #self.ssl = False + self.verbose_default = 1 + self.timeout_default = None + + def add_options(self): + super(PostgreSQLCLI, self).add_options() + self.add_hostoption() + self.add_useroption() + self.add_opt('-d', '--database', help='Database to connect to') + + def process_options(self): + super(PostgreSQLCLI, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.user = self.user + self.password = self.password + #self.ssl = self.get_opt('ssl') + + def connect(self, database): + log.info('connecting to %s:%s database %s as user %s', self.host, self.port, database) + return psycopg2.connect(host=self.host, + port=self.port, + database=database, + user=self.user, + password=self.password) From e61f6e95c76be66d24e4844676c93e04131b4d9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 11:31:57 +0000 Subject: [PATCH 0622/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index da235c81a..56c669530 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 200+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies * [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... From adc6d6a431c5e6cba8e489ecd84f960cccd527e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:04:09 +0000 Subject: [PATCH 0623/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f72e25d7e..50b4cf805 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f72e25d7eb699c38d248525c36197f6d99bb96d2 +Subproject commit 50b4cf805992083e8a675a41a979db56e5814901 From 0e4be2ff7f07c0580bbfa9a11d87ea540c778050 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:04:10 +0000 Subject: [PATCH 0624/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7de1ce1dd..01d8135a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7de1ce1ddd165b8f91b285df5cce9a49fbba7b42 +Subproject commit 01d8135a535d15a52aadd3c9419130a32e75fb25 From 58e39ede54ad4c90ac214a935c37cb66106cd1dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:40:09 +0000 Subject: [PATCH 0625/2295] updated postgres_cli.py --- lib/postgres_cli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py index 68864934f..7ccd5b617 100755 --- a/lib/postgres_cli.py +++ b/lib/postgres_cli.py @@ -27,7 +27,7 @@ sys.path.append(libdir) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_host, validate_port + from harisekhon.utils import log, validate_host, validate_port, validate_user, validate_password from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -66,12 +66,14 @@ def add_options(self): def process_options(self): super(PostgreSQLCLI, self).process_options() self.host = self.get_opt('host') - self.port = self.get_opt('port') + self.host = self.get_opt('host') + self.user = self.get_opt('user') + self.password = self.get_opt('password') validate_host(self.host) validate_port(self.port) + validate_user(self.user) + validate_password(self.password) self.port = int(self.port) - self.user = self.user - self.password = self.password #self.ssl = self.get_opt('ssl') def connect(self, database): From 9293a92f99f82d59c630e4b6736f9770640b3381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:06:35 +0000 Subject: [PATCH 0626/2295] updated requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5df77ddb1..9187ac1c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,9 @@ ldif3==3.2.2 # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 -psycopg2==2.8.4 +# requires pg_config to build from source +#psycopg2==2.8.4 +psycopg2-binary==2.8.4 python-cson==1.0.9 psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From 0026f3fcc09728faf6e88a4894ca19b6a7f364ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:16:20 +0000 Subject: [PATCH 0627/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 50b4cf805..e6161d191 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 50b4cf805992083e8a675a41a979db56e5814901 +Subproject commit e6161d19183fa3e9e71613c523add85ba35cfb06 From 341958ffa970c5889c6341fc7303ecf08a227576 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:16:20 +0000 Subject: [PATCH 0628/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 01d8135a5..1439449f0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 01d8135a535d15a52aadd3c9419130a32e75fb25 +Subproject commit 1439449f000f62886b5e69387f789a6be9fce08c From 4a21e96e579dca84b4944fd53823126af6c427bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 21:39:07 +0000 Subject: [PATCH 0629/2295] added .buildkite/pipeline.yml --- .buildkite/pipeline.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..25a2cdc35 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,37 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# BuildKite Pipeline +# +# add this command to the UI and it will read the rest of the steps from here: +# +# - command: buildkite-agent pipeline upload + +steps: + - command: ./install_packages.sh make + label: install make + timeout: 10 + - wait + - command: make init + label: init + timeout: 2 + - wait + - command: make ci + label: build + timeout: 60 + - wait + - command: make test + label: test + timeout: 120 From 01c9a656fca176738f9112030697b8c8b98aac43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:39:30 +0000 Subject: [PATCH 0630/2295] updated pipeline.yml --- .buildkite/pipeline.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 25a2cdc35..c8bfbe881 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -20,9 +20,22 @@ # - command: buildkite-agent pipeline upload steps: - - command: ./install_packages.sh make + - command: | + type make 2>/dev/null || + if type apk 2>/dev/null; then + apk add --no-cache --no-progress make + # apt is /usr/bin/apt + # Unable to locate an executable at "/Users/hari/.sdkman/candidates/java/current/bin/apt" (-1) + elif type apt-get 2>/dev/null; then + apt-get update -q && + apt-get install -qy make + elif type yum 2>/dev/null; then + rpm -q make || yum install -y make + elif type brew 2>/dev/null; then + brew install make + fi label: install make - timeout: 10 + timeout: 20 # brew can take 10 mins just to do a brew update - wait - command: make init label: init From 8f628028196b1cd6100e58a1338ea27bf32ef7f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:48:13 +0000 Subject: [PATCH 0631/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 56c669530..5bd9249db 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Hari Sekhon - DevOps Python Tools [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 1f13192c74407d58c590e81f834ed9dc76e4faa2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:49:08 +0000 Subject: [PATCH 0632/2295] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index 67e6f895d..efef4f8a5 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # openldap-dev +postgresql-dev snappy-dev # installed by bash-tools submodule now From d1eebf208ed61a1b8186bdb1c4266e631b37bc5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:51:52 +0000 Subject: [PATCH 0633/2295] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 6dd8fa477..2537f9e1a 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel +libpq-devel # postgres pg_config openldap-devel openssl-devel From 77f8db87e2a110bc40b9bff095252469271a7162 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:52:28 +0000 Subject: [PATCH 0634/2295] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index c956b3189..1a879d376 100644 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -26,4 +26,7 @@ python2-psutil #python-markupsafe #python2-bitarray +python2-psycopg2 +python3-psycopg2 + python3-snappy From 6eb2369a6668d3dab74e2d26c9810a9eef1b69a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:54:01 +0000 Subject: [PATCH 0635/2295] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index a2e311a6b..ae1a456ce 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # libldap2-dev +libpq-dev # postgres pg_config # needed to build python-snappy for avro module libsnappy-dev From 1383a2b084071abc025290a9b552cc5f1bf46595 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:54:58 +0000 Subject: [PATCH 0636/2295] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 75196cfb9..8c6215957 100644 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -24,6 +24,7 @@ python-ldap python-ldif3 python-numpy python-psutil +python-psycopg2 python-sh python-snappy python-thrift From 64ec4628ed120a102c4b4a72fcf949180c83e9b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:55:44 +0000 Subject: [PATCH 0637/2295] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index 0ab2bc769..01130d497 100644 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -20,3 +20,4 @@ py3-jinja2 py3-numpy #py3-flask py3-pygit2 +py3-psycopg2 From ced5c3ace5bdba2906a063502487a1ad361c7e91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:57:29 +0000 Subject: [PATCH 0638/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e6161d191..7b9ecd187 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e6161d19183fa3e9e71613c523add85ba35cfb06 +Subproject commit 7b9ecd187bf79e02f996915c4a606bbfa9da19e8 From 17983db85203d0badd94a5f430b82644904d1c8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:57:29 +0000 Subject: [PATCH 0639/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1439449f0..fe836adba 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1439449f000f62886b5e69387f789a6be9fce08c +Subproject commit fe836adba0c49c3a0b3316318653200469dcb6e1 From ff41881b970f837bca5592782aed6359c3406653 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Mar 2020 16:15:01 +0000 Subject: [PATCH 0640/2295] updated validate_json.py --- validate_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_json.py b/validate_json.py index 03cc0aa7a..1d83b8b04 100755 --- a/validate_json.py +++ b/validate_json.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.11.1' +__version__ = '0.11.2' class JsonValidatorTool(CLI): @@ -210,7 +210,7 @@ def check_json(self, content): return True self.failed = True if not self.passthru: - die(self.self.invalid_json_msg_single_quotes) + die(self.invalid_json_msg_single_quotes) else: log.debug('not valid json') if self.rewind_check_multirecord_json(): From 73e6506c2c87aa88c6ef0db9b0078bc054e2d9b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Mar 2020 21:29:51 +0000 Subject: [PATCH 0641/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 345 ++++++++++++++++++------------ 1 file changed, 210 insertions(+), 135 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index dc647a528..fee6feb28 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -16,7 +16,7 @@ """ -Processes Cloudera Navigator exported CSV logs to list the tables used (selected from) +Processes Cloudera Navigator API exported CSV logs to list the tables used (selected from) This allows you to see if you wasting time maintaining datasets nobody is using @@ -25,15 +25,16 @@ 1. table/database name fields are often blank and need to be inferred from SQL queries field 2. SQL queries often contain newlines which break the rows up 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used + 4. optionally ignore users by regex, matching user or kerberos principal to omit ETL service account -See cloudera_navigator_audit_download_logs.sh for a script to export these logs +See cloudera_navigator_audit_logs_download.sh for a script to export these logs -./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \ +./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\ navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv -Output - CSV format to stdout: +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): -database,table +"database","table" Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 @@ -45,7 +46,7 @@ #from __future__ import unicode_literals import csv -#import logging +import logging import os import re import sys @@ -56,7 +57,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log + from harisekhon.utils import CriticalError, log, validate_regex from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -65,7 +66,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class ClouderaNavigatorTablesUsed(CLI): @@ -78,8 +79,31 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None - self.data = {} + #self.data = {} + self.indicies = {} + self.len_headers = None + self.operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + 'USE', + ] self.timeout_default = None + self.table_regex = r'[\w\.`]+' + self.re_table = re.compile(self.table_regex) + self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ + .format(table_regex=self.table_regex), \ + re.I | re.MULTILINE | re.DOTALL) + self.re_ignore = re.compile(r'\b(?:SHOW|DESCRIBE|USE|REFRESH|INVALIDATE\S+METADATA|GET_TABLES|GET_SCHEMAS)\b',\ + re.I | re.MULTILINE | re.DOTALL) + # 2020-01-31T20:45:59.000Z + self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') + self.re_ignored_users = None + self.csv_writer = None def add_options(self): super(ClouderaNavigatorTablesUsed, self).add_options() @@ -91,12 +115,20 @@ def add_options(self): self.add_opt('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV output (recommended, default is double quote \'"\')') self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)') + self.add_opt('-i', '--ignore-users', help='Users to ignore, comma separated regex values') def process_options(self): super(ClouderaNavigatorTablesUsed, self).process_options() self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') + ignore_users = self.get_opt('ignore_users') + if ignore_users: + ignored_users = ignore_users.split(',') + for username in ignored_users: + validate_regex(username, 'ignored user') + # account for kerberized names - user, user@domain.com or user/host@domain.com + self.re_ignored_users = re.compile('^' + '|'.join(ignored_users) + '(?:@|/|$)', re.I) if not self.args: self.usage('no CSV file argument given') @@ -106,23 +138,30 @@ def run(self): quoting = csv.QUOTE_NONE fieldnames = ['database', 'table'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=self.delimiter, - quotechar=self.quotechar, - escapechar=self.escapechar, - quoting=quoting, - fieldnames=fieldnames) + self.csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) for filename in self.args: - self.process_file(filename) + with open(filename, 'rU') as filehandle: + self.process_file(filehandle) - csv_writer.writeheader() - for database in sorted(self.data): - for table in sorted(self.data[database]): - csv_writer.writerow({'database': database, - 'table': table}) + #csv_writer.writeheader() + #for database in sorted(self.data): + # for table in sorted(self.data[database]): + # csv_writer.writerow({'database': database, + # 'table': table}) #if log.isEnabledFor(logging.DEBUG): # sys.stdout.flush() +# Navigator API Audit log output is a mess with duplicate columns and different naming conventions, +# eg. identical SQL in fields 18 and 36 +# table and database names in fields 19+21 vs 40+41 +# all duplicates with different header names +# +# # same result for navigator_audit_2019_impala.csv # csv_header_indices.sh navigator_audit_2019_hive.csv # 0 Timestamp # 1 Username @@ -169,125 +208,161 @@ def run(self): # 42 "Object Type" # 43 Privilege - # TODO: should really be refactored to be smaller simpler chunks of code - # XXX: this post processing is ugly as hell and probably brittle - YMMV - def process_file(self, filename): - re_select_from_table = re.compile(r'\bselect\b.+\bfrom\b(?:\s|\n)+([^\s,]+)', re.I | re.MULTILINE | re.DOTALL) - operations_to_ignore = [ - '', - 'HIVEREPLICATIONCOMMAND', - 'START', - 'STOP', - 'RESTART', - 'LOAD', - 'SWITCHDATABASE', - ] - with open(filename) as csvfile: - csv_reader = csv.reader(csvfile, delimiter=',', quotechar='"', escapechar='\\') - headers = csv_reader.next() - len_headers = len(headers) - # needed to ensure row joining works later on with number of fields left - assert len_headers == 44 - operation_index = 4 - table_index = 19 - database_index = 21 - sql_index = 36 - assert headers[operation_index] == 'Operation' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' - assert headers[sql_index] == 'Operation Text' - partial_row = [] - sql_decomment = self.sql_decomment - # more complicated than I wish it was - msg me if you know a simpler cleaner way - for row in csv_reader: - #log.debug('row = %s', row) - #try: - # various logic to handle rows broken on newlines inside SQL queries - len_row = len(row) - if len_row > len_headers: - #log.debug('collapsing fields in row: %s', row) - difference = len_row - len_headers - row[sql_index] = ','.join([sql_decomment(_) for _ in row[sql_index:difference]]) - row = row[:sql_index] + row[sql_index + difference:] - len_row = len(row) - #log.debug('collapsed row: %s', row) - #log.debug('row length: %s', len_row) - #log.debug('partial row length: %s', len(partial_row)) - if len_row == len_headers: + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 44 + user_index = 1 + operation_index = 4 + resource_index = 5 + table_index = 19 + database_index = 21 + # fields 18 and 36 are identical SQL - need both to collapse rows later + sql_index = 18 + sql_index2 = 36 + assert headers[user_index] == 'Username' + assert headers[operation_index] == 'Operation' + assert headers[resource_index] == 'Resource' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'operation_text' + assert headers[sql_index2] == 'Operation Text' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation + } + self.process_rows(csv_reader) + + # logic to reconstruct rows because the Navigator API breaks the record format + # with newlines in SQL coming out literally and fragmenting the records + def process_rows(self, csv_reader): + last_row = [] + for current_row in csv_reader: + if not current_row: + continue + if self.re_timestamp.match(current_row[0]): + row = last_row + last_row = current_row + else: + last_row += current_row + continue + if not row: + continue + self.process_row(row) + self.process_row(last_row) + + def process_row(self, row): + log.debug('row = %s', row) + len_row = len(row) + if len_row > self.len_headers: + row = self.collapse_sql_fields(row=row) + len_row = len(row) + if len_row != self.len_headers: + raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + .format(len_row, self.len_headers, row)) + (database, table) = self.parse_table(row) + self.output(row=row, database=database, table=table) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # 'hari.sekhon' in 'hari.sekhon@somedomain.com' in kerberos + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: pass - elif len_row < len_headers: - log.debug('row (partial): %s', row) - if len_row + len(partial_row) == len_headers + 1: - #log.debug('length row + partial_row == header length, completing partial row') - #log.debug('partial_row = %s', partial_row) - #log.debug('row = %s', row) - # join first field to last field to complete SQL query - sql_fragment = sql_decomment(row[0]) - partial_row[-1] = partial_row[-1] + r'\n ' + sql_fragment - partial_row += row[1:] - #log.debug('partial_row = %s', partial_row) - elif partial_row: - #log.debug('partial_row: %s', partial_row) - #log.debug('row: %s', row) - # join next fragment of SQL query to incomplete last item containing the first part of SQL query - partial_row[-1] = partial_row[-1] + r'\n ' + r'\n '.join(row) - #log.debug('accumulated partial row: %s', partial_row) - elif len(partial_row) > len_headers: - raise CriticalError('len(partial_row) > len_headers - {} > {} for partial row: {}'\ - .format(len(partial_row), len_headers, partial_row)) - else: - partial_row = row - #log.debug('partial_row = %s', partial_row) - #log.debug('len partial row = %s', len(partial_row)) - #log.debug('len headers = %s', len_headers) - if len(partial_row) == len_headers: - # process accumulated row as normal - row = partial_row - partial_row = [] - #log.debug('accumulated completed row: %s', row) - else: - continue - elif partial_row: - raise CriticalError('incompleted partial row: {}'.format(partial_row)) - len_row = len(row) - if len_row != len_headers: - raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ - .format(len_row, len_headers, row)) - #log.debug(row) - database = row[21] - table = row[19] - if not table.strip(): - operation = row[4] - if operation == 'QUERY': - log.debug('table not found in row: %s', row) - query = row[36] - log.debug('trying to parse: %s', query) - match = re_select_from_table.search(query) - if match: - table = match.group(1) - if '.' in table: - (database, table) = table.split('.', 1) - else: - log.warning('failed to parse table from query: %s', query) - elif operation in operations_to_ignore: - continue + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) else: - log.debug('table not found in row and operation is not a query to parse: %s', row) - if not table and not database: - continue - table = table.lower() - database = database.lower() - self.data[database] = self.data.get(database, {}) - self.data[database][table] = 1 - #except IndexError as _: -# if log.isEnabledFor(logging.DEBUG): -# log.error('%s - offending line: %s', _, row) -# else: - # raise CriticalError('ERROR: %s - offending line: %s', _, row) + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + table = table.lower().strip('`') + database = database.lower().strip('`') + if ' ' in table: + raise CriticalError('table "{}"'.format(table)) + if ' ' in database: + raise CriticalError('database "{}"'.format(database)) + return (database, table) + + def get_db_table_from_resource(self, row): + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource: + # database:table in Resource field + (database, table) = resource.split(':', 1) + return (database, table) + + def output(self, row, database, table): + if not self.re_table.match('{}.{}'.format(database, table)): + log.warning('%s.%s does not match table regex', database, table) + return + #self.data[database] = self.data.get(database, {}) + #self.data[database][table] = 1 + if table and not database: + log.info('got table but not database for row: %s', row) + if database and not table: + log.info('got database but not table for row: %s', row) + if not table and not database: + return + self.csv_writer.writerow({'database': database, 'table': table}) + if log.isEnabledFor(logging.DEBUG): + sys.stdout.flush() + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + sql_index2 = self.indicies['sql_index2'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + difference = len_row - self.len_headers + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) + row = row[:sql_index2] + row[sql_index2+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row @staticmethod def sql_decomment(string): - return string.split('--')[0] + return string.split('--')[0].strip() if __name__ == '__main__': From 33af4d3b176df8500852a2b096b3647e7428bbfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Mar 2020 14:13:12 +0000 Subject: [PATCH 0642/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 39 ++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index fee6feb28..2918ce8b3 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -16,16 +16,20 @@ """ -Processes Cloudera Navigator API exported CSV logs to list the tables used (selected from) +Processes Cloudera Navigator API exported CSV logs to list the tables used (SELECT'ed from) -This allows you to see if you wasting time maintaining datasets nobody is using +This allows you to see if you're wasting time maintaining datasets nobody is using Handles more than naive filtering delimited column numbers which will miss many table and database names: 1. table/database name fields are often blank and need to be inferred from SQL queries field - 2. SQL queries often contain newlines which break the rows up - 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used - 4. optionally ignore users by regex, matching user or kerberos principal to omit ETL service account + 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records + 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used + 4. where table/database field aren't available, also checks resource field for suitable contents as another heuristic + to determine database and table name before parsing SQL which is a last resort + 5. optionally ignore selected users by regex + - matches user or kerberos principal + - eg. to omit ETL service account from skewing data access results See cloudera_navigator_audit_logs_download.sh for a script to export these logs @@ -57,7 +61,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log, validate_regex + from harisekhon.utils import CriticalError, log, validate_regex, isInt from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -128,7 +132,7 @@ def process_options(self): for username in ignored_users: validate_regex(username, 'ignored user') # account for kerberized names - user, user@domain.com or user/host@domain.com - self.re_ignored_users = re.compile('^' + '|'.join(ignored_users) + '(?:@|/|$)', re.I) + self.re_ignored_users = re.compile('^(?:' + '|'.join(ignored_users) + ')(?:[@/]|$)', re.I) if not self.args: self.usage('no CSV file argument given') @@ -247,6 +251,7 @@ def process_rows(self, csv_reader): for current_row in csv_reader: if not current_row: continue + # originally did this by counting fields but SQL fragmentation generates extra fields if self.re_timestamp.match(current_row[0]): row = last_row last_row = current_row @@ -259,6 +264,8 @@ def process_rows(self, csv_reader): self.process_row(last_row) def process_row(self, row): + if not row: + return log.debug('row = %s', row) len_row = len(row) if len_row > self.len_headers: @@ -314,16 +321,20 @@ def parse_table(self, row): table = table.lower().strip('`') database = database.lower().strip('`') if ' ' in table: - raise CriticalError('table "{}"'.format(table)) + raise CriticalError('table \'{}\' has spaces - parsing error for row: {}'.format(table, row)) if ' ' in database: - raise CriticalError('database "{}"'.format(database)) + raise CriticalError('database \'{}\' has spaces - parsing error for row: {}'.format(database, row)) + if table == 'null': + raise CriticalError('table == null - parsing error for row: {}'.format(row)) return (database, table) def get_db_table_from_resource(self, row): database = None table = None resource = row[self.indicies['resource_index']] - if resource: + if resource and \ + ':' in resource and \ + 'null:null' not in resource: # database:table in Resource field (database, table) = resource.split(':', 1) return (database, table) @@ -350,7 +361,13 @@ def collapse_sql_fields(self, row): len_row = len(row) if len_row > self.len_headers: log.debug('collapsing fields in row: %s', row) - difference = len_row - self.len_headers + # divide by 2 to account for this having been done twice in duplicated SQL operational text + difference = (len_row - self.len_headers) / 2 + # slice indicies must be integers + if not isInt(difference): + raise CriticalError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) row = row[:sql_index] + row[sql_index+difference:] row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) From 87d6ba94dfc191266e02948824c47c8dafba36a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Mar 2020 16:35:35 +0000 Subject: [PATCH 0643/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 966142672..53b219274 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -491,6 +491,9 @@ dest[141]="Failed to open HDFS file hdfs:///user//warehouse/ Date: Wed, 18 Mar 2020 16:36:09 +0000 Subject: [PATCH 0644/2295] added API token stripping --- anonymize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 94beaf3f6..8e84d0c5f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.10' +__version__ = '0.10.11' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -377,6 +377,9 @@ def __init__(self): sep=arg_sep, pass_word_phrase=pass_word_phrase, pw=password_quoted), + 'password4': r'([\.-]?(?:api-?)?token{sep}){pw}'\ + .format(sep=arg_sep, + pw=password_quoted), 'ip': r'(?', 'password2': r'\1:', 'password3': r'\1\2', + 'password4': r'\1', 'ip': r'/', 'ip2': r'', 'ip3': r'', From 98f0914624d46b3619ce0cde6cab8465e2033ec6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 10:36:29 +0000 Subject: [PATCH 0645/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5bd9249db..542df1894 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Hari Sekhon - DevOps Python Tools [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![All Code](https://img.shields.io/badge/all%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) From 3ff8c1371d1935a10e60a3443202f7964eeaa7ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 10:40:39 +0000 Subject: [PATCH 0646/2295] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 542df1894..680feb726 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) + +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) From 0c99607be2e13c8245536910054be6ffb3c1d97a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 12:07:55 +0000 Subject: [PATCH 0647/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 84 ++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 2918ce8b3..eb1cee773 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -23,9 +23,10 @@ Handles more than naive filtering delimited column numbers which will miss many table and database names: 1. table/database name fields are often blank and need to be inferred from SQL queries field + (currently limited to 'SELECT ... FROM ...' because JOINs are often complicated by use of table aliases) 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used - 4. where table/database field aren't available, also checks resource field for suitable contents as another heuristic + 4. where table/database field aren't available, also checks if inferrable from resource field to determine database and table name before parsing SQL which is a last resort 5. optionally ignore selected users by regex - matches user or kerberos principal @@ -41,6 +42,7 @@ "database","table" Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 +(but may require ongoing tweaks depending on quirks in your data set or changes in the API / logs) """ @@ -61,7 +63,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log, validate_regex, isInt + from harisekhon.utils import log, validate_regex, isInt from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -99,6 +101,7 @@ def __init__(self): self.timeout_default = None self.table_regex = r'[\w\.`]+' self.re_table = re.compile(self.table_regex) + # doesn't handle JOINs because SQL pros usually use table aliases self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ .format(table_regex=self.table_regex), \ re.I | re.MULTILINE | re.DOTALL) @@ -221,18 +224,29 @@ def process_file(self, filehandle): user_index = 1 operation_index = 4 resource_index = 5 - table_index = 19 - database_index = 21 + object_index = 22 # used by collapse_sql_fields to check if SQL was split, do not change to index 33! + # -- + # with massive queries taking the latter 2 is more likely to succeed, + # possibly because there is a rare and subtle issue in collapse_sql_fields + #table_index = 19 + #database_index = 21 + # or + table_index = 41 + database_index = 40 + # -- # fields 18 and 36 are identical SQL - need both to collapse rows later sql_index = 18 sql_index2 = 36 + #assert headers[table_index] == 'table_name' # index 19 + #assert headers[database_index] == 'database_name' # index 21 + assert headers[table_index] == 'Table Name' # index 41 + assert headers[database_index] == 'Database Name' # index 40 assert headers[user_index] == 'Username' assert headers[operation_index] == 'Operation' assert headers[resource_index] == 'Resource' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' assert headers[sql_index] == 'operation_text' assert headers[sql_index2] == 'Operation Text' + assert headers[object_index] == 'object_type' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -241,6 +255,7 @@ def process_file(self, filehandle): 'database_index': database_index, 'sql_index': sql_index, 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation + 'object_index': object_index } self.process_rows(csv_reader) @@ -249,6 +264,7 @@ def process_file(self, filehandle): def process_rows(self, csv_reader): last_row = [] for current_row in csv_reader: + #log.debug('current row = %s', current_row) if not current_row: continue # originally did this by counting fields but SQL fragmentation generates extra fields @@ -266,13 +282,14 @@ def process_rows(self, csv_reader): def process_row(self, row): if not row: return - log.debug('row = %s', row) + log.debug('processing row = %s', row) len_row = len(row) + log.debug('row len = %s', len_row) if len_row > self.len_headers: row = self.collapse_sql_fields(row=row) len_row = len(row) if len_row != self.len_headers: - raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + raise AssertionError('row items ({}) != header items ({}) for offending row: {}'\ .format(len_row, self.len_headers, row)) (database, table) = self.parse_table(row) self.output(row=row, database=database, table=table) @@ -318,14 +335,18 @@ def parse_table(self, row): return (None, None) if not table and not database: return (None, None) - table = table.lower().strip('`') - database = database.lower().strip('`') - if ' ' in table: - raise CriticalError('table \'{}\' has spaces - parsing error for row: {}'.format(table, row)) - if ' ' in database: - raise CriticalError('database \'{}\' has spaces - parsing error for row: {}'.format(database, row)) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) if table == 'null': - raise CriticalError('table == null - parsing error for row: {}'.format(row)) + raise AssertionError('table == null - parsing error for row: {}'.format(row)) return (database, table) def get_db_table_from_resource(self, row): @@ -343,6 +364,8 @@ def output(self, row, database, table): if not self.re_table.match('{}.{}'.format(database, table)): log.warning('%s.%s does not match table regex', database, table) return + # instead of collecting in ram, now just post-process through sort -u + # this way it is easier to see live extractions, --debug and correlate #self.data[database] = self.data.get(database, {}) #self.data[database][table] = 1 if table and not database: @@ -358,18 +381,29 @@ def output(self, row, database, table): def collapse_sql_fields(self, row): sql_index = self.indicies['sql_index'] sql_index2 = self.indicies['sql_index2'] + object_index = self.indicies['object_index'] len_row = len(row) if len_row > self.len_headers: log.debug('collapsing fields in row: %s', row) # divide by 2 to account for this having been done twice in duplicated SQL operational text - difference = (len_row - self.len_headers) / 2 - # slice indicies must be integers - if not isInt(difference): - raise CriticalError("difference in field length '{}' is not an integer for row: {}"\ - .format(difference, row)) - difference = int(difference) - row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) - row = row[:sql_index] + row[sql_index+difference:] + # Update: appears this broke as only 2nd occurence of SQL operational text field got split to new fields, + # which is weird because the log shows both 1st and 2nd SQL text fields were double quoted + difference = len_row - self.len_headers + # seems first occurrence doesn't get split in some occurence, + # wasn't related to open in newline universal mode though + # if 2 fields after isn't the /user/hive/warehouse/blah.db then 1st SQL wasn't split + # would have to regex /user/hive/warehouse/blah.db(?:/table)? + #if not row[sql_index+2].endswith('.db'): + # if object field is TABLE or DATABASE then 1st sql field wasn't split + if row[object_index] not in ('TABLE', 'DATABASE'): + difference /= 2 + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) row = row[:sql_index2] + row[sql_index2+difference:] log.debug('collapsed row: %s', row) @@ -381,6 +415,10 @@ def collapse_sql_fields(self, row): def sql_decomment(string): return string.split('--')[0].strip() + @staticmethod + def index_output(obj): + return '\n'.join(['{}\t{}'.format(index, item) for (index, item) in enumerate(obj)]) + if __name__ == '__main__': ClouderaNavigatorTablesUsed().main() From 45cfdfcae980202d45836cf8a94dd3bdeeb53765 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:16:35 +0000 Subject: [PATCH 0648/2295] updated travis_last_log.py --- travis_last_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_last_log.py b/travis_last_log.py index 5005c3417..c703ef300 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class TravisLastBuildLog(CLI): @@ -151,7 +151,7 @@ def process_options(self): self.repo = self.get_local_repo_name() if not self.repo: self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) self.num = self.get_opt('num') validate_int(self.num, 'num', 1) From 281b32e1eb6ecadd1d75d12cb7db8fc87e44fe65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:16:45 +0000 Subject: [PATCH 0649/2295] updated travis_debug_session.py --- travis_debug_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index eb2557d7d..805fecdd0 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -69,7 +69,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class TravisDebugSession(CLI): @@ -155,7 +155,7 @@ def process_options(self): self.repo = self.get_local_repo_name() if not self.repo: self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) @staticmethod From 436cf262a6862cce124d820c67c09daba350c2ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:17:38 +0000 Subject: [PATCH 0650/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7b9ecd187..bcf1929dd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7b9ecd187bf79e02f996915c4a606bbfa9da19e8 +Subproject commit bcf1929ddb1bf016b6c29d02e7482eb003735c74 From eeb390d1f6b20e1fe0edd879d04c1b3c0a7f4a81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:17:39 +0000 Subject: [PATCH 0651/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fe836adba..13d728ab8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fe836adba0c49c3a0b3316318653200469dcb6e1 +Subproject commit 13d728ab831def658188c07d4130074ef0cfd214 From 94caafbc5ac3b9871c519b1c436b92a48c1bb29f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:17:23 +0000 Subject: [PATCH 0652/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 43 +++++++++++++++++++------------ 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index eb1cee773..57902b25e 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -52,6 +52,7 @@ #from __future__ import unicode_literals import csv +import gzip import logging import os import re @@ -72,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' class ClouderaNavigatorTablesUsed(CLI): @@ -85,20 +86,10 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None + self.timeout_default = None #self.data = {} self.indicies = {} self.len_headers = None - self.operations_to_ignore = [ - '', - 'HIVEREPLICATIONCOMMAND', - 'START', - 'STOP', - 'RESTART', - 'LOAD', - 'SWITCHDATABASE', - 'USE', - ] - self.timeout_default = None self.table_regex = r'[\w\.`]+' self.re_table = re.compile(self.table_regex) # doesn't handle JOINs because SQL pros usually use table aliases @@ -111,6 +102,16 @@ def __init__(self): self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.re_ignored_users = None self.csv_writer = None + self.operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + 'USE', + ] def add_options(self): super(ClouderaNavigatorTablesUsed, self).add_options() @@ -152,8 +153,12 @@ def run(self): quoting=quoting, fieldnames=fieldnames) for filename in self.args: - with open(filename, 'rU') as filehandle: - self.process_file(filehandle) + if filename.endswith('.gz'): + with gzip.open(filename, 'rU') as filehandle: + self.process_file(filehandle) + else: + with open(filename, 'rU') as filehandle: + self.process_file(filehandle) #csv_writer.writeheader() #for database in sorted(self.data): @@ -267,8 +272,7 @@ def process_rows(self, csv_reader): #log.debug('current row = %s', current_row) if not current_row: continue - # originally did this by counting fields but SQL fragmentation generates extra fields - if self.re_timestamp.match(current_row[0]): + if self.is_new_record(current_row): row = last_row last_row = current_row else: @@ -279,6 +283,10 @@ def process_rows(self, csv_reader): self.process_row(row) self.process_row(last_row) + # originally did this by counting fields but SQL fragmentation generates extra fields + def is_new_record(self, current_row): + return self.re_timestamp.match(current_row[0]) + def process_row(self, row): if not row: return @@ -297,7 +305,8 @@ def process_row(self, row): def parse_table(self, row): #log.debug(row) user = row[self.indicies['user_index']] - # 'hari.sekhon' in 'hari.sekhon@somedomain.com' in kerberos + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' if self.re_ignored_users and self.re_ignored_users.match(user): log.debug('skipping row for ignored user %s: %s', user, row) return (None, None) From ae413f806d031b092d6e6448d05f6a069c7040f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:27:10 +0000 Subject: [PATCH 0653/2295] added cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 218 +++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100755 cloudera_navigator_tables_used_postgres.py diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py new file mode 100755 index 000000000..7de004292 --- /dev/null +++ b/cloudera_navigator_tables_used_postgres.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Processes Cloudera Navigator CSV logs exported from PostgreSQL to list the tables used (SELECT'ed from) + +This allows you to see if you're wasting time maintaining datasets nobody is using + +See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs + +Supports reading directly from gzipped logs if they end in .gz file extension + +./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_2019_11_*.csv.gz \\ + nav.public.impala_audit_events_2019_11_*.csv.gz ... + +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): + +"database","table" + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +#import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +import gzip +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, isInt + #from harisekhon import CLI + from cloudera_navigator_tables_used import ClouderaNavigatorTablesUsed +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsedPostgreSQL, self).__init__() + # Python 3.x + # super().__init__() + # recombine records due to SQL \n breaking up records, new records start like: + # 306529,1574163624392,1,hive, + self.re_new_record = re.compile(r'^\d+,\d+,[01],(?:hive|impala),') + # get db + table from resource path (just one layer of checks) + self.re_resource = re.compile(r'/(\w+)\.db/(\w+)') + +# Navigator table logs: +# +# gzcat nav.public.hive_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 ip_addr +# 6 operation +# 7 database_name +# 8 object_type +# 9 table_name +# 10 operation_text +# 11 impersonator +# 12 resource_path +# 13 object_usage_type + + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 14 + user_index = 4 + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[user_index] == 'username' + assert headers[operation_index] == 'operation' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'operation_text' + assert headers[resource_index] == 'resource_path' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + } + self.process_rows(csv_reader) + + def is_new_record(self, current_row): + return self.re_new_record.match(','.join(current_row)) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: + pass + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) + else: + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) + if table == 'null': + raise AssertionError('table == null - parsing error for row: {}'.format(row)) + return (database, table) + + def get_db_table_from_resource(self, row): + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource: + match = self.re_resource.search(resource) + if match: + database = match.group(1) + table = match.group(2) + return (database, table) + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + difference = len_row - self.len_headers + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsedPostgreSQL().main() From a3f598c5373e3e6c848127b2450d2f481f7bbbd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:43:12 +0000 Subject: [PATCH 0654/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 57902b25e..616e6de72 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -96,7 +96,17 @@ def __init__(self): self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ .format(table_regex=self.table_regex), \ re.I | re.MULTILINE | re.DOTALL) - self.re_ignore = re.compile(r'\b(?:SHOW|DESCRIBE|USE|REFRESH|INVALIDATE\S+METADATA|GET_TABLES|GET_SCHEMAS)\b',\ + ignore_statements = [ + 'SHOW', + 'DESCRIBE', + 'USE', + 'REFRESH', + r'INVALIDATE\S+METADATA', + 'GET_TABLES', + 'GET_SCHEMAS', + 'VIEW_METADATA', + ] + self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + ')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') From e01917971d5888ce89cd01d6a3b14d57fdc53e9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:54:34 +0000 Subject: [PATCH 0655/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 616e6de72..01297e695 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -32,6 +32,9 @@ - matches user or kerberos principal - eg. to omit ETL service account from skewing data access results +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. + See cloudera_navigator_audit_logs_download.sh for a script to export these logs ./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\ @@ -164,9 +167,11 @@ def run(self): fieldnames=fieldnames) for filename in self.args: if filename.endswith('.gz'): + log.debug("processing gzip'd file: %s", filename) with gzip.open(filename, 'rU') as filehandle: self.process_file(filehandle) else: + log.debug("processing file: %s", filename) with open(filename, 'rU') as filehandle: self.process_file(filehandle) From 7c47723e338798083072e7c445add7d57f42a67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:12:19 +0000 Subject: [PATCH 0656/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 01297e695..f479ca06f 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -67,7 +67,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_regex, isInt + from harisekhon.utils import log, validate_regex, isInt, isPythonMinVersion from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -86,6 +86,7 @@ def __init__(self): super(ClouderaNavigatorTablesUsed, self).__init__() # Python 3.x # super().__init__() + csv.field_size_limit(sys.maxsize) self.delimiter = None self.quotechar = None self.escapechar = None @@ -109,7 +110,7 @@ def __init__(self): 'GET_SCHEMAS', 'VIEW_METADATA', ] - self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + ')\b',\ + self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + r')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') @@ -165,14 +166,17 @@ def run(self): escapechar=self.escapechar, quoting=quoting, fieldnames=fieldnames) + mode = 'rtU' + if isPythonMinVersion(3): + mode = 'rt' for filename in self.args: if filename.endswith('.gz'): log.debug("processing gzip'd file: %s", filename) - with gzip.open(filename, 'rU') as filehandle: + with gzip.open(filename, mode, encoding="utf8") as filehandle: self.process_file(filehandle) else: log.debug("processing file: %s", filename) - with open(filename, 'rU') as filehandle: + with open(filename, mode, encoding="utf8") as filehandle: self.process_file(filehandle) #csv_writer.writeheader() @@ -237,7 +241,12 @@ def run(self): def process_file(self, filehandle): csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') - headers = csv_reader.next() + try: + # Python 2 + headers = csv_reader.next() + except AttributeError: + # Python 3 + headers = next(csv_reader) self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 44 From a84227d8b72030a87e4369c87e3cd64b445be533 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:18:16 +0000 Subject: [PATCH 0657/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index f479ca06f..f1c66c350 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -169,14 +169,15 @@ def run(self): mode = 'rtU' if isPythonMinVersion(3): mode = 'rt' + # open(..., encoding="utf-8") is Python 3 only - uses system default otherwise for filename in self.args: if filename.endswith('.gz'): log.debug("processing gzip'd file: %s", filename) - with gzip.open(filename, mode, encoding="utf8") as filehandle: + with gzip.open(filename, mode) as filehandle: self.process_file(filehandle) else: log.debug("processing file: %s", filename) - with open(filename, mode, encoding="utf8") as filehandle: + with open(filename, mode) as filehandle: self.process_file(filehandle) #csv_writer.writeheader() From be910fc932ead11113061e4169f94b9d2e3172b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:36:01 +0000 Subject: [PATCH 0658/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index f1c66c350..6f73105b2 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -104,13 +104,20 @@ def __init__(self): 'SHOW', 'DESCRIBE', 'USE', - 'REFRESH', - r'INVALIDATE\S+METADATA', + 'CREATE', + 'DROP', + 'INSERT', + 'DELETE', + 'UPDATE', 'GET_TABLES', 'GET_SCHEMAS', 'VIEW_METADATA', + 'ANALYZE', + r'COMPUTE\s+STATS' + 'REFRESH', + r'INVALIDATE\s+METADATA', ] - self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + r')\b',\ + self.re_ignore = re.compile(r'^\s*\b(?:' + '|'.join(ignore_statements) + r')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') From b06c6f1ba6a7557aa2ac2239b52ca4fcf88a2908 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:40:14 +0000 Subject: [PATCH 0659/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bcf1929dd..621e7e2fd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bcf1929ddb1bf016b6c29d02e7482eb003735c74 +Subproject commit 621e7e2fdf660122d94549a5630e518fee66cf3d From 02fe7a948abe24fce5be4c04a630cb8e43f8f2d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:41:07 +0000 Subject: [PATCH 0660/2295] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 69 ++++++++++++++++------ 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7de004292..7c994b529 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -22,10 +22,11 @@ See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs -Supports reading directly from gzipped logs if they end in .gz file extension +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. -./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_2019_11_*.csv.gz \\ - nav.public.impala_audit_events_2019_11_*.csv.gz ... +./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_*.csv.gz \\ + nav.public.impala_audit_events_*.csv.gz ... Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): @@ -63,7 +64,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): @@ -97,24 +98,55 @@ def __init__(self): # 12 resource_path # 13 object_usage_type +# gzcat nav.public.impala_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 impersonator +# 6 ip_addr +# 7 operation +# 8 query_id +# 9 session_id +# 10 status +# 11 database_name +# 12 object_type +# 13 table_name +# 14 privilege +# 15 operation_text + def process_file(self, filehandle): csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') headers = csv_reader.next() self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left - assert self.len_headers == 14 - user_index = 4 - operation_index = 6 - database_index = 7 - table_index = 9 - sql_index = 10 - resource_index = 12 - assert headers[user_index] == 'username' - assert headers[operation_index] == 'operation' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' - assert headers[sql_index] == 'operation_text' - assert headers[resource_index] == 'resource_path' + assert self.len_headers == 14 or self.len_headers == 16 + # Hive postgres audit log + if self.len_headers == 14: + user_index = 4 + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[user_index] == 'username' + assert headers[resource_index] == 'resource_path' + # Impala postgres audit log + elif self.len_headers == 16: + user_index = 5 # impersonator field contains actual user, user field is always 'impala' + operation_index = 7 + database_index = 11 + table_index = 13 + sql_index = 15 + resource_index = None + assert headers[user_index] == 'impersonator' + else: + raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') + assert headers[sql_index] == 'operation_text' + assert headers[database_index] == 'database_name' + assert headers[table_index] == 'table_name' + assert headers[operation_index] == 'operation' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -185,6 +217,9 @@ def parse_table(self, row): return (database, table) def get_db_table_from_resource(self, row): + # only available for hive audit logs, not impala + if self.indicies['resource_index'] is None: + return (None, None) database = None table = None resource = row[self.indicies['resource_index']] From 091495164b13dbfadced36ee2913ea798db73e45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 16:24:48 +0000 Subject: [PATCH 0661/2295] updated aws_users_last_used.py --- aws_users_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 588a1d7af..3c467292b 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -67,7 +67,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class AWSUsersLastUsed(CLI): @@ -112,7 +112,7 @@ def run(self): filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) assert headers[0] == 'user' assert headers[4] == 'password_last_used' assert headers[10] == 'access_key_1_last_used_date' From cb28e00d175c5052ee4c0a19e61b5baa0a41e10d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 16:24:54 +0000 Subject: [PATCH 0662/2295] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 9b9083eb9..fcb244e07 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -65,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' class AWSUnusedAccessKeys(CLI): @@ -113,7 +113,7 @@ def run(self): filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) log.debug('headers: %s', headers) assert headers[0] == 'user' assert headers[8] == 'access_key_1_active' From eda0798c7920ee8d923e6c4a77d2b22f0a820aed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:34:27 +0000 Subject: [PATCH 0663/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 6f73105b2..cbd95d6da 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -166,6 +166,7 @@ def run(self): if self.quotechar == '': quoting = csv.QUOTE_NONE + #fieldnames = ['database', 'table', 'user'] fieldnames = ['database', 'table'] self.csv_writer = csv.DictWriter(sys.stdout, delimiter=self.delimiter, @@ -415,6 +416,7 @@ def output(self, row, database, table): log.info('got database but not table for row: %s', row) if not table and not database: return + #self.csv_writer.writerow({'database': database, 'table': table, 'user': row[self.indicies['user_index']]}) self.csv_writer.writerow({'database': database, 'table': table}) if log.isEnabledFor(logging.DEBUG): sys.stdout.flush() From 6db6bf7e60cd4b7173feed8b283d3b7e29b0cd44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:34:35 +0000 Subject: [PATCH 0664/2295] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7c994b529..7d2fcaee9 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -122,25 +122,23 @@ def process_file(self, filehandle): self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 14 or self.len_headers == 16 + user_index = 4 + assert headers[user_index] == 'username' # Hive postgres audit log if self.len_headers == 14: - user_index = 4 operation_index = 6 database_index = 7 table_index = 9 sql_index = 10 resource_index = 12 - assert headers[user_index] == 'username' assert headers[resource_index] == 'resource_path' # Impala postgres audit log elif self.len_headers == 16: - user_index = 5 # impersonator field contains actual user, user field is always 'impala' operation_index = 7 database_index = 11 table_index = 13 sql_index = 15 resource_index = None - assert headers[user_index] == 'impersonator' else: raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') assert headers[sql_index] == 'operation_text' From 8fa7e4c15e7d469b8356939c0d7a736d17851afb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:37:40 +0000 Subject: [PATCH 0665/2295] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index cbd95d6da..645d344b4 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -113,7 +113,7 @@ def __init__(self): 'GET_SCHEMAS', 'VIEW_METADATA', 'ANALYZE', - r'COMPUTE\s+STATS' + r'COMPUTE\s+STATS', 'REFRESH', r'INVALIDATE\s+METADATA', ] From a6391e6e34b6030ba8d9a6fb5e085519f76e288c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Mar 2020 13:45:50 +0000 Subject: [PATCH 0666/2295] added .concourse.yml --- .concourse.yml | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .concourse.yml diff --git a/.concourse.yml b/.concourse.yml new file mode 100644 index 000000000..d50f7ecf4 --- /dev/null +++ b/.concourse.yml @@ -0,0 +1,59 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +resources: +- name: github + icon: github-circle + type: git + source: + uri: https://github.com/harisekhon/devops-python-tools + branch: master +#- name: daily +# type: time +# source: +# interval: 1d + +# https://concourse-ci.org/golang-library-example.html + +jobs: +- name: build + public: false + plan: + - get: github # from resource above + trigger: true + #version: every # build every git commit, default: latest + - task: build + config: + platform: linux + image_resource: + type: docker-image + source: + repository: ubuntu + tag: latest + inputs: + - name: github + path: code + params: + CONCOURSE: 1 + run: + path: /bin/bash + args: + - -c + - | + cd code && + apt update -q && + apt install -qy git make && + make init && + make ci test From e84131dd049aeec5618db68da2ca4d6fcb3337e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:15:36 +0000 Subject: [PATCH 0667/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 680feb726..5770ebfe3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Hari Sekhon - DevOps Python Tools [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 9594dcbd63893015af7dc2438e2b0c037e752600 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:20:59 +0000 Subject: [PATCH 0668/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5770ebfe3..957b93c43 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Hari Sekhon - DevOps Python Tools [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) From cfbb467e9de520257831ea9494f25f334485b37a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:28:15 +0000 Subject: [PATCH 0669/2295] updated aws_users_last_used.py --- aws_users_last_used.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 3c467292b..9a93b7bad 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -54,7 +54,6 @@ from io import StringIO from math import floor import boto3 -from botocore.exceptions import ClientError srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) From d8821908b574b8637799eca69de74559d3d511f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:34:21 +0000 Subject: [PATCH 0670/2295] updated dockerhub_show_tags.py --- dockerhub_show_tags.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py index 88c9ddc08..e0dfecbb8 100755 --- a/dockerhub_show_tags.py +++ b/dockerhub_show_tags.py @@ -32,6 +32,7 @@ import json import logging import os +import re import sys import traceback import urllib @@ -52,7 +53,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class DockerHubTags(CLI): @@ -79,8 +80,11 @@ def run(self): self.usage('no repos given as args') self.quiet = self.get_opt('quiet') if not self.quiet: + # cheaper but lgtm hassling me, not a security issue but will shut them up print('\nDocker', end='') - if 'registry.hub.docker.com' in self.url_base: + #if 'registry.hub.docker.com' in self.url_base: + # match anchors but I prefer explicit anchor, more intuitive for other generic language coders + if re.match(r'^https://registry\.hub\.docker\.com/', self.url_base): print('Hub') else: print(' Registry: {0}'.format(self.url_base.split('/v2', 1)[0])) From 208b3d82cf74ae30bebe01c5ed391776ec7f632d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:37:44 +0000 Subject: [PATCH 0671/2295] updated dockerfiles_check_git_branches.py --- dockerfiles_check_git_branches.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py index a7edf5195..b2e407f4b 100755 --- a/dockerfiles_check_git_branches.py +++ b/dockerfiles_check_git_branches.py @@ -91,7 +91,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.7.3' class DockerfileGitBranchCheckTool(CLI): @@ -321,8 +321,8 @@ def check_file(self, filename, branch): def check_dockerfile_arg(self, filename, branch): log.debug('check_dockerfile_arg({0}, {1})'.format(filename, branch)) - branch_base = str(branch).replace('-dev', '') - (branch_base, branch_versions) = self.branch_version(branch) + branch_stripped = str(branch).replace('-dev', '') + (branch_base, branch_versions) = self.branch_version(branch_stripped) with open(filename) as filehandle: version_index = 0 for line in filehandle: From 62215f42f321b2a6c622ba241dbe3775b5417f1c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:38:31 +0000 Subject: [PATCH 0672/2295] updated anonymize.py --- anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 8e84d0c5f..418427711 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.11' +__version__ = '0.10.12' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -219,7 +219,7 @@ def __init__(self): #'countryCode', 'displayName', 'displayNamePrintable', - 'division' + 'division', 'employeeID', 'groupMembershipSAM', 'info', From 2e6c324f451ff2e483db95cd18ea7ac2c9ca89ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:49:22 +0000 Subject: [PATCH 0673/2295] refactored out generate sql static method for overriding --- hive_tables_null_rows.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 7ab8b53dd..82082d358 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -62,7 +62,7 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class HiveTablesNullRows(HiveForEachTable): @@ -79,11 +79,11 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, _query): + def execute(self, conn, database, table, _): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec + # impala library doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) column_cursor.execute('use `{}`'.format(database)) @@ -92,9 +92,7 @@ def execute(self, conn, database, table, _query): column = column_row[0] #column_type = column_row[1] columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL AND `".join(columns) + "` IS NULL" + query = self.generate_sql(database, table, columns) with conn.cursor() as table_cursor: log.debug('executing query: %s', query) table_cursor.execute(query) @@ -102,6 +100,13 @@ def execute(self, conn, database, table, _query): count = result[0] print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + @staticmethod + def generate_sql(database, table, columns): + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL AND `".join(columns) + "` IS NULL" + return sql + if __name__ == '__main__': HiveTablesNullRows().main() From f08bb8cdca5db6869cf28c682e86a3f27eeadf21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:50:19 +0000 Subject: [PATCH 0674/2295] deduped code using inheriting overriding of hive_tables_null_rows --- hive_tables_row_counts_any_nulls.py | 54 +++++++---------------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index ce17ca99a..a1de5e461 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -52,8 +52,7 @@ sys.path.append(pylib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from hive_foreach_table import HiveForEachTable + from hive_tables_null_rows import HiveTablesNullRows except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -62,45 +61,18 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' - - -class HiveTablesRowsWithNulls(HiveForEachTable): - - def __init__(self): - # Python 2.x - super(HiveTablesRowsWithNulls, self).__init__() - # Python 3.x - # super().__init__() - self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option - self.database = None - self.table = None - #self.partition = None - self.ignore_errors = False - - # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, _query): - columns = [] - log.info("describing table '%s.%s'", database, table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use `{}`'.format(database)) - column_cursor.execute('describe `{}`'.format(table)) - for column_row in column_cursor: - column = column_row[0] - #column_type = column_row[1] - columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL OR `".join(columns) + "` IS NULL" - with conn.cursor() as table_cursor: - log.debug('executing query: %s', query) - table_cursor.execute(query) - for result in table_cursor: - count = result[0] - print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) +__version__ = '0.6.0' + + +class HiveTablesRowsWithNulls(HiveTablesNullRows): + + @staticmethod + def generate_sql(database, table, columns): + # impala library doesn't support parameterized query quoting from dbapi spec + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL OR `".join(columns) + "` IS NULL" + return sql if __name__ == '__main__': From 8fb1e49f80a32690a98741d04b7fc477ade72873 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:54:05 +0000 Subject: [PATCH 0675/2295] updated anonymize.py --- anonymize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/anonymize.py b/anonymize.py index 418427711..d46b6c796 100755 --- a/anonymize.py +++ b/anonymize.py @@ -72,18 +72,20 @@ # pylint: disable=unused-import from harisekhon.utils import \ aws_host_ip_regex, \ - domain_regex, \ domain_regex_strict, \ - email_regex, \ filename_regex, \ fqdn_regex, \ host_regex, \ hostname_regex, \ ip_prefix_regex, \ ip_regex, \ - mac_regex, \ subnet_mask_regex, \ user_regex + from harisekhon.utils import \ + domain_regex, \ + email_regex, \ + mac_regex \ + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') From cb43cbb372912c4c11cf6a18c21b8d52511855af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:59:25 +0000 Subject: [PATCH 0676/2295] updated validate_avro.py --- validate_avro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_avro.py b/validate_avro.py index f8a896466..f81c3afb7 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -39,7 +39,7 @@ try: from avro3.datafile import DataFileReader, DataFileException from avro3.io import DatumReader -except: +except Exception: # pylint: disable=broad-except from avro.datafile import DataFileReader, DataFileException from avro.io import DatumReader libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) @@ -55,7 +55,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class AvroValidatorTool(CLI): From f1f2ee3f4bf2aa93ef58b07c7d38bb0527503f6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:00:29 +0000 Subject: [PATCH 0677/2295] updated hbase_flush_tables.py --- hbase_flush_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index c4f43ef8d..04d51bc75 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -44,7 +44,7 @@ import sys import traceback import subprocess -from subprocess import PIPE +PIPE = subprocess.PIPE libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: From 71df727098d340a786154f2435f51add299a27e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:02:54 +0000 Subject: [PATCH 0678/2295] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 955b395e7..efcbc1279 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -60,11 +60,11 @@ from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToAvro(CLI): From 20e0d0c4d6b73840405a2388d6ec65a2839651f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:03:15 +0000 Subject: [PATCH 0679/2295] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index f3f6909e6..3bbb9c18a 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -52,11 +52,11 @@ from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToParquet(CLI): From a541399757edfd051a50eda9be2703d1b82a960b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:03:46 +0000 Subject: [PATCH 0680/2295] updated validate_multimedia.py --- validate_multimedia.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validate_multimedia.py b/validate_multimedia.py index 9452fd6c7..54def6deb 100755 --- a/validate_multimedia.py +++ b/validate_multimedia.py @@ -45,7 +45,7 @@ import re import sys import subprocess -from subprocess import CalledProcessError +CalledProcessError = subprocess.CalledProcessError libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: @@ -164,8 +164,8 @@ def check_path(self, path): die("failed to determine if path '%s' is file or directory" % path) def check_media_file(self, filename): - if self.is_excluded(filename): - return + #if self.is_excluded(filename): + # return valid_media_msg = '%s => OK' % filename invalid_media_msg = '%s => INVALID' % filename cmd = self.validate_cmd From 071afdd59ed494b7fdd1990dc28fcbfb6b550f6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:49:58 +0000 Subject: [PATCH 0681/2295] added buddy.yml --- buddy.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 buddy.yml diff --git a/buddy.yml b/buddy.yml new file mode 100644 index 000000000..9b897a700 --- /dev/null +++ b/buddy.yml @@ -0,0 +1,38 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://buddy.works/docs/yaml/yaml-schema + +- pipeline: "Build" + trigger_mode: "ON_EVERY_PUSH" + ref_name: "master" + ref_type: "BRANCH" + target_site_url: "https://github.com/harisekhon/devops-python-tools" + trigger_condition: "ALWAYS" + actions: + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/devops-python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + execute_commands: + - "apt update &&" + - "apt install -qy make &&" + - "make init &&" + - "make ci test" + volume_mappings: + - "/:/buddy/devops-python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" From b764c9f6cd49011023128ab3884fcba8f9a83efd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:05 +0000 Subject: [PATCH 0682/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 621e7e2fd..da52b0d76 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 621e7e2fdf660122d94549a5630e518fee66cf3d +Subproject commit da52b0d76dc4d0db3189f668ea3b01ccc735ac3c From f692ec345f1c53894ec5bb67d573df0501e9b18c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:05 +0000 Subject: [PATCH 0683/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 13d728ab8..c657aea1e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 13d728ab831def658188c07d4130074ef0cfd214 +Subproject commit c657aea1e99f9f811faa9aa4980b44153664e0c3 From 0bd5cff97e6b56d4259bbe7f26d9875e3d9637b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:28 +0000 Subject: [PATCH 0684/2295] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 957b93c43..4b1e73070 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,10 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) -[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) +[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.concourse.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 9b1bf6d09a89cdd47e98c1c41b033d629f91596f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:54:15 +0000 Subject: [PATCH 0685/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b1e73070..1b1d18109 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Hari Sekhon - DevOps Python Tools [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.concourse.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From db6782f56c4e2236f14d56d485f1e4bdd06ddb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 02:01:14 +0000 Subject: [PATCH 0686/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b1d18109..cc3c36f59 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) From 86dfaa8a6f4af4a57e1fb5e4f2ea908cc4c41d8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 02:02:16 +0000 Subject: [PATCH 0687/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc3c36f59..3afe9935f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) From f23adf59b72d97f5a78a237096ab14b0a1245beb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 11:39:42 +0000 Subject: [PATCH 0688/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da52b0d76..16b436757 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da52b0d76dc4d0db3189f668ea3b01ccc735ac3c +Subproject commit 16b4367579ff71056b3bdfb62b19550bf334c193 From 04c5217e1a298755a0e8aab7d89149b74bf7b4dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 11:39:42 +0000 Subject: [PATCH 0689/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c657aea1e..cd591772c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c657aea1e99f9f811faa9aa4980b44153664e0c3 +Subproject commit cd591772cb701831e207fa253aef56920ca7fc8d From 643c1c31819d98186804501f061beb49614cf39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:32 +0000 Subject: [PATCH 0690/2295] added gocd_config_repo.json --- setup/gocd_config_repo.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 setup/gocd_config_repo.json diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json new file mode 100644 index 000000000..a3a8beb73 --- /dev/null +++ b/setup/gocd_config_repo.json @@ -0,0 +1,26 @@ +{ + "id": "devops-python-tools", + "plugin_id": "yaml.config.plugin", + "material": { + "type": "git", + "attributes": { + "url": "https://github.com/harisekhon/devops-python-tools", + "branch": "master", + "auto_update": true + } + }, + "configuration": [ + { + "key": "file_pattern", + "value": "*.gocd.y*ml" + } + ], + "rules": [ + { + "directive": "allow", + "action": "*", + "type": "*", + "resource": "*" + } + ] +} From 0cd3ced174bcb1e702648fad23b21f8851482618 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:35 +0000 Subject: [PATCH 0691/2295] added .gocd.yml --- .gocd.yml | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .gocd.yml diff --git a/.gocd.yml b/.gocd.yml new file mode 100644 index 000000000..ad1c3d693 --- /dev/null +++ b/.gocd.yml @@ -0,0 +1,83 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://github.com/tomzo/gocd-yaml-config-plugin#setup + +# https://docs.gocd.org/current/configuration/configuration_reference.html + +format_version: 3 +pipelines: + devops-python-tools: + group: defaultGroup + label_template: ${COUNT} + lock_behavior: none + display_order: -1 + materials: + git: + git: https://github.com/harisekhon/devops-python-tools + shallow_clone: false + auto_update: true + branch: master + stages: + - build-and-test: + fetch_materials: true + keep_artifacts: false + clean_workspace: false + approval: + type: success + allow_only_on_success: false + jobs: + apt-update: + timeout: 10 + tasks: + - exec: + command: apt + arguments: + - update + run_if: passed + install-make: + timeout: 10 + tasks: + - exec: + command: apt + arguments: + - install + - -qy + - git + - make + run_if: passed + init: + timeout: 10 + tasks: + - exec: + command: make + arguments: + - init + run_if: passed + build: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - ci + run_if: passed + test: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - test + run_if: passed From 841b8126493d9ad1e334fac0213c54751a4c1807 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:57 +0000 Subject: [PATCH 0692/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3afe9935f..f5b5d1519 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Hari Sekhon - DevOps Python Tools [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 544a5a4d4f0dbe53b66b4277a22cd5cfcee7a6a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:12:32 +0000 Subject: [PATCH 0693/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 16b436757..2f73faf5a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 16b4367579ff71056b3bdfb62b19550bf334c193 +Subproject commit 2f73faf5a9f2afc9c535858ed2c987c12b176306 From 2312e6d4a6759d494ffd1207f87877103ccda137 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:12:32 +0000 Subject: [PATCH 0694/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cd591772c..2ed6c0629 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cd591772cb701831e207fa253aef56920ca7fc8d +Subproject commit 2ed6c06298ccd2421e8c8177d5ffa93321dc76ae From 1737491ca62dcfc03b35723de122c8217f412aba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:24 +0000 Subject: [PATCH 0695/2295] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 55 +++++++++++----------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7d2fcaee9..47bf1b419 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -49,7 +49,6 @@ srcdir = os.path.abspath(os.path.dirname(__file__)) pylib = os.path.join(srcdir, 'pylib') lib = os.path.join(srcdir, 'lib') -import gzip sys.path.append(pylib) sys.path.append(lib) try: @@ -64,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): @@ -122,29 +121,29 @@ def process_file(self, filehandle): self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 14 or self.len_headers == 16 - user_index = 4 - assert headers[user_index] == 'username' - # Hive postgres audit log - if self.len_headers == 14: - operation_index = 6 - database_index = 7 - table_index = 9 - sql_index = 10 - resource_index = 12 - assert headers[resource_index] == 'resource_path' - # Impala postgres audit log - elif self.len_headers == 16: - operation_index = 7 - database_index = 11 - table_index = 13 - sql_index = 15 - resource_index = None - else: - raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') - assert headers[sql_index] == 'operation_text' - assert headers[database_index] == 'database_name' - assert headers[table_index] == 'table_name' - assert headers[operation_index] == 'operation' + user_index = 4 + assert headers[user_index] == 'username' + # Hive postgres audit log + if self.len_headers == 14: + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[resource_index] == 'resource_path' + # Impala postgres audit log + elif self.len_headers == 16: + operation_index = 7 + database_index = 11 + table_index = 13 + sql_index = 15 + resource_index = None + else: + raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') + assert headers[sql_index] == 'operation_text' + assert headers[database_index] == 'database_name' + assert headers[table_index] == 'table_name' + assert headers[operation_index] == 'operation' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -215,9 +214,9 @@ def parse_table(self, row): return (database, table) def get_db_table_from_resource(self, row): - # only available for hive audit logs, not impala - if self.indicies['resource_index'] is None: - return (None, None) + # only available for hive audit logs, not impala + if self.indicies['resource_index'] is None: + return (None, None) database = None table = None resource = row[self.indicies['resource_index']] From e3979c89f52b16fd955c6d09b20d24bcd904fae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:31 +0000 Subject: [PATCH 0696/2295] updated .appveyor.yml --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c1e4e06ac..a97f97008 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,10 +28,10 @@ install: # The following packages have unmet dependencies: # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. -# bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed # make[2]: *** [apt-packages] Error 123 # make[2]: Leaving directory '/home/appveyor/projects/pylib' -# bash-tools/Makefile.in:212: recipe for target 'system-packages' failed +# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed # # adding "|| :" to the end of these commands causes them to be silently ignored! - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list From 50b24cbf567353f2691ac3ef3d15c361cfb4fe80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:35 +0000 Subject: [PATCH 0697/2295] updated shippable.yml --- shippable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 6b45ef91f..43f564753 100644 --- a/shippable.yml +++ b/shippable.yml @@ -29,7 +29,7 @@ build: # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed + # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make From 8241e557a6315ec6d313374938140ab86d2b1578 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 04:55:57 +0100 Subject: [PATCH 0698/2295] added Jenkinsfile --- Jenkinsfile | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..0c1855ec4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,134 @@ +#!/usr/bin/env groovy +// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy +// +// Author: Hari Sekhon +// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) +// +// https://github.com/harisekhon/devops-python-tools +// +// License: see accompanying Hari Sekhon LICENSE file +// +// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +// +// https://www.linkedin.com/in/harisekhon +// + +// ========================================================================== // +// J e n k i n s P i p e l i n e +// ========================================================================== // + + +// https://jenkins.io/doc/book/pipeline/syntax/ + + +pipeline { + // run pipeline any agent + agent any + // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' +// agent { +// docker { +// image 'ubuntu:18.04' +// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' +// } +// } + + // need to specify at least one env var if enabling + //environment { + // DEBUG = '1' + //} + + options { + // put timestamps in console logs + timestamps() + + // timeout entire pipeline after 4 hours + timeout(time: 4, unit: 'HOURS') + + //retry entire pipeline 3 times + //retry(3) + } + + triggers { + cron('H 10 * * 1-5') + pollSCM('H/2 * * * *') + } + + stages { + stage ('Checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + } + } + + stage('Build') { + steps { + echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" + echo 'Building...' + timeout(time: 10, unit: 'MINUTES') { + retry(3) { +// sh 'apt update -q' +// sh 'apt install -qy make' +// sh 'make init' + sh """ + apt update -q && + apt install -qy make && + make init + """ + } + } + timeout(time: 180, unit: 'MINUTES') { + sh 'make ci' + } + } + } + + stage('Test') { + options { + retry(2) + } + steps { + echo 'Testing...' + timeout(time: 120, unit: 'MINUTES') { + sh 'make test' + } + } + } + +// stage('Human gate') { +// steps { +// input "Proceed to deployment?" +// } +// } +// +// stage('Deployment') { +// steps { +// echo 'Deploying...' +// echo 'Nothing to deploy' +// } +// } + + } + post { + always { + echo 'Always' + //deleteDir() // clean up workspace + + // collect JUnit reports for Jenkins UI + //junit 'build/reports/**/*.xml' + // collect artifacts to Jenkins for analysis + //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true + } + success { + echo 'SUCCESS!' + } + failure { + echo 'FAILURE!' + } + unstable { + echo 'UNSTABLE!' + } + changed { + echo 'Pipeline state change! (success vs failure)' + } + } +} From 3be0fa7b8d271587c5eed03bd72d6b2d251202fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 04:55:59 +0100 Subject: [PATCH 0699/2295] added jenkins-job.xml --- setup/jenkins-job.xml | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 setup/jenkins-job.xml diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml new file mode 100644 index 000000000..4469d2afe --- /dev/null +++ b/setup/jenkins-job.xml @@ -0,0 +1,57 @@ + + + + + + + + hudson.triggers.SCMTrigger + hudson.triggers.TimerTrigger + + + + + + + false + + + + https://github.com/harisekhon/devops-python-tools/ + + + + + + H 10 * * 1-5 + + + H/2 * * * * + false + + + + + + + 2 + + + https://github.com/harisekhon/devops-python-tools + + + + + */master + + + false + + + + Jenkinsfile + true + + + false + \ No newline at end of file From a51f3b0ec279bb9549c6c791a02583d5833ee34c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:02:13 +0100 Subject: [PATCH 0700/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f5b5d1519..380e9e7cd 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Hari Sekhon - DevOps Python Tools [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From f4e31a54a4626d54078d5d2f7d96839215f48acb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:03:13 +0100 Subject: [PATCH 0701/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2f73faf5a..450fe18ff 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2f73faf5a9f2afc9c535858ed2c987c12b176306 +Subproject commit 450fe18ff0341e348235645bb6cdaedcee2c7b00 From 47388a28394faed0b4dfd9f2678212ae51743069 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:03:13 +0100 Subject: [PATCH 0702/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2ed6c0629..22de6ec06 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2ed6c06298ccd2421e8c8177d5ffa93321dc76ae +Subproject commit 22de6ec0636ddbb61e4b710d2d32a0a4c6a37775 From 6c11f3653036719d2c11598f0136c61d6cc70968 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:00 +0100 Subject: [PATCH 0703/2295] updated Jenkinsfile --- Jenkinsfile | 56 ++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0c1855ec4..c63e3e540 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,35 +22,35 @@ pipeline { - // run pipeline any agent + // run pipeline any agent agent any - // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' + // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' // agent { -// docker { -// image 'ubuntu:18.04' -// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' -// } -// } +// docker { +// image 'ubuntu:18.04' +// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' +// } +// } - // need to specify at least one env var if enabling + // need to specify at least one env var if enabling //environment { - // DEBUG = '1' - //} + // DEBUG = '1' + //} options { // put timestamps in console logs timestamps() - // timeout entire pipeline after 4 hours - timeout(time: 4, unit: 'HOURS') + // timeout entire pipeline after 4 hours + timeout(time: 4, unit: 'HOURS') - //retry entire pipeline 3 times - //retry(3) + //retry entire pipeline 3 times + //retry(3) } - triggers { + triggers { cron('H 10 * * 1-5') - pollSCM('H/2 * * * *') + pollSCM('H/2 * * * *') } stages { @@ -69,11 +69,11 @@ pipeline { // sh 'apt update -q' // sh 'apt install -qy make' // sh 'make init' - sh """ - apt update -q && - apt install -qy make && - make init - """ + sh """ + apt update -q && + apt install -qy make && + make init + """ } } timeout(time: 180, unit: 'MINUTES') { @@ -103,20 +103,20 @@ pipeline { // stage('Deployment') { // steps { // echo 'Deploying...' -// echo 'Nothing to deploy' +// echo 'Nothing to deploy' // } // } } - post { + post { always { echo 'Always' - //deleteDir() // clean up workspace + //deleteDir() // clean up workspace - // collect JUnit reports for Jenkins UI - //junit 'build/reports/**/*.xml' - // collect artifacts to Jenkins for analysis - //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true + // collect JUnit reports for Jenkins UI + //junit 'build/reports/**/*.xml' + // collect artifacts to Jenkins for analysis + //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true } success { echo 'SUCCESS!' From baf63b3b5affb9ae1614a47893df298ed015dbb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:44 +0100 Subject: [PATCH 0704/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 450fe18ff..3b044f028 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 450fe18ff0341e348235645bb6cdaedcee2c7b00 +Subproject commit 3b044f02840047b21931ecb54ab50e72fcf51efd From c7b27780b3a6a2adc02e3279a3dcb70968ea67c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:44 +0100 Subject: [PATCH 0705/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 22de6ec06..ac2c5362a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 22de6ec0636ddbb61e4b710d2d32a0a4c6a37775 +Subproject commit ac2c5362a27e7f77f823a60bdfb6e3277b32b66d From d9f2c4e90dd4d247913e060bf814da330ba8c1ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:25:42 +0100 Subject: [PATCH 0706/2295] updated rpm-packages-optional.txt --- setup/rpm-packages-optional.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt index 11e757a6f..cb889fb12 100644 --- a/setup/rpm-packages-optional.txt +++ b/setup/rpm-packages-optional.txt @@ -19,3 +19,9 @@ yamllint snappy-devel # CentOS 8 csnappy-devel + +# postgres pg_config +# CentOS 8 +libpq-devel +# CentOS 7 +postgresql-devel From 5378204d20bd4b0bd7e675a89f8e2d9b43efca9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:25:50 +0100 Subject: [PATCH 0707/2295] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 2537f9e1a..6dd8fa477 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,7 +17,6 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel -libpq-devel # postgres pg_config openldap-devel openssl-devel From 6a5de18e631bb58b42468b3f4efc61557e2e1074 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:33:37 +0100 Subject: [PATCH 0708/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3b044f028..19b964eef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3b044f02840047b21931ecb54ab50e72fcf51efd +Subproject commit 19b964eef3f2d6a95f13ca8bfc7b4b598e55d1ea From f4b325e0ab68cd420f32720460a3b3c8a46c61b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:33:37 +0100 Subject: [PATCH 0709/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ac2c5362a..289fce9b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ac2c5362a27e7f77f823a60bdfb6e3277b32b66d +Subproject commit 289fce9b55e7d8464f62d20e106dc88f71165e03 From 9d427b5dce3393bc1586bc12e2aee416b84911da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:35:18 +0100 Subject: [PATCH 0710/2295] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 380e9e7cd..e9a8ab490 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,11 @@ Hari Sekhon - DevOps Python Tools [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) --> [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) + + [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 5a124d955410c1b84053a27806d0645a3d5f1488 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 14:49:58 +0100 Subject: [PATCH 0711/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 19b964eef..079f5cd34 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 19b964eef3f2d6a95f13ca8bfc7b4b598e55d1ea +Subproject commit 079f5cd34b88c6a8c6f07bcd011e68fee5879ea0 From 11bc05c086ab03d65fd3ccac22aa8fb9351911f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 14:49:58 +0100 Subject: [PATCH 0712/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 289fce9b5..9974c54ca 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 289fce9b55e7d8464f62d20e106dc88f71165e03 +Subproject commit 9974c54cab38f6d9f8b5250a6f02f50e7e058444 From 99ae9d0468803c7eab00dedc3e0448c77e035b9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:01:36 +0100 Subject: [PATCH 0713/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 079f5cd34..d29d724e2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 079f5cd34b88c6a8c6f07bcd011e68fee5879ea0 +Subproject commit d29d724e230fb3403982b805f29dd22ea807e3d3 From c4a70bdb0912043290d8ec9b229240fdd97cf6e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:01:36 +0100 Subject: [PATCH 0714/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9974c54ca..8bd7ad714 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9974c54cab38f6d9f8b5250a6f02f50e7e058444 +Subproject commit 8bd7ad714a45d19d44ebe40eaf4bb6bd8b426d06 From 853b054feedceda1ba950422b0bf26d2d895fa0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:40:54 +0100 Subject: [PATCH 0715/2295] updated Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index c3689b47e..e9823683c 100755 --- a/Makefile +++ b/Makefile @@ -57,6 +57,8 @@ build: @echo ========================= @echo DevOps Python Tools Build @echo ========================= + @bash-tools/git_summary_line.sh + @echo @# executing in sh where type is not available @#type -P python From a31c1c90be15b08c05cc1c9509e9c9bf8c178da1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:41:03 +0100 Subject: [PATCH 0716/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d29d724e2..100b4e6c6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d29d724e230fb3403982b805f29dd22ea807e3d3 +Subproject commit 100b4e6c65d877e136df18b484e2f7848b9f4c45 From aa29b6641713c397e2be06d4b8c73abe5f57394e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:41:03 +0100 Subject: [PATCH 0717/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8bd7ad714..c264da75c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8bd7ad714a45d19d44ebe40eaf4bb6bd8b426d06 +Subproject commit c264da75cfd484753a712fece7acc1405e1b8309 From 0387479ed6d3f0b2138bd2b928d3687a05aac23a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:34 +0100 Subject: [PATCH 0718/2295] updated Makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e9823683c..3b5d29391 100755 --- a/Makefile +++ b/Makefile @@ -53,11 +53,11 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: +build: init @echo ========================= @echo DevOps Python Tools Build @echo ========================= - @bash-tools/git_summary_line.sh + @$(MAKE) git-summary @echo @# executing in sh where type is not available @@ -66,7 +66,6 @@ build: python -V || : pip -V || : - $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi From 1d102fc1bfb6fc821e054c0599b6e44016849be1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:50 +0100 Subject: [PATCH 0719/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 100b4e6c6..f60229172 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 100b4e6c65d877e136df18b484e2f7848b9f4c45 +Subproject commit f60229172e123257e3798549348c9079591cbe0e From 55215d0a8ce06ebc3940d6c70a6bfab8513ee33b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:50 +0100 Subject: [PATCH 0720/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c264da75c..4e7bbd2ac 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c264da75cfd484753a712fece7acc1405e1b8309 +Subproject commit 4e7bbd2ace765b28deef6e8b20ec3c76680190e2 From 2d1023350d9d02d0784a00cb1f05984824ea5973 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:02:33 +0100 Subject: [PATCH 0721/2295] updated Makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 3b5d29391..4eb3ff5e1 100755 --- a/Makefile +++ b/Makefile @@ -58,7 +58,6 @@ build: init @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary - @echo @# executing in sh where type is not available @#type -P python From 1132d7c8f61c7743584e29fbdbaaa9e2741f6868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 21:44:01 +0100 Subject: [PATCH 0722/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f60229172..80a090777 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f60229172e123257e3798549348c9079591cbe0e +Subproject commit 80a090777285e0b22c1225756a2e07323625b8a3 From 5055dd80fc33380a62a09547b9e537f9ed835868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 21:44:01 +0100 Subject: [PATCH 0723/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4e7bbd2ac..5c9b49fbe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4e7bbd2ace765b28deef6e8b20ec3c76680190e2 +Subproject commit 5c9b49fbe3e28f606fb6dc62e023894fa9cae236 From 829cde2bb53a92d3bf7dc698c78f3cda21f64782 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 1 Apr 2020 14:03:58 +0100 Subject: [PATCH 0724/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 80a090777..167c12679 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 80a090777285e0b22c1225756a2e07323625b8a3 +Subproject commit 167c12679aa4c5779e989c9bfc41ef605ff1ab3f From 2eb77b0e1662f73e3feeaf82d3cabd40f17ae17a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 1 Apr 2020 14:03:58 +0100 Subject: [PATCH 0725/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5c9b49fbe..aead6c401 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5c9b49fbe3e28f606fb6dc62e023894fa9cae236 +Subproject commit aead6c401b0c7ec92366eee33cce338f8eb6a8cb From c2cb9ea9f428f2ed275e3699b35c9beb0bdcc119 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:06:20 +0100 Subject: [PATCH 0726/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 167c12679..d67dcede2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 167c12679aa4c5779e989c9bfc41ef605ff1ab3f +Subproject commit d67dcede2bbff47072b655ee641357723d91cc0f From 5e2e95b0753c5c85e0e2a874fbaa8e20c51b65d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:06:20 +0100 Subject: [PATCH 0727/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aead6c401..0d1827a52 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aead6c401b0c7ec92366eee33cce338f8eb6a8cb +Subproject commit 0d1827a52e68a4174e8c1059971e884e2c14cb67 From a6643ab4f5fe3e09b9cf34b8df60565f5a05cf5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:31:57 +0100 Subject: [PATCH 0728/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9a8ab490..0ffbe1aa5 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STATUS.md) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) From 65a40cd1b1b4e0f82ee958970ce98f3b78d709d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:05:13 +0100 Subject: [PATCH 0729/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0ffbe1aa5..4490e2f81 100644 --- a/README.md +++ b/README.md @@ -441,3 +441,5 @@ You might also be interested in the following really nice Jupyter notebook for H ### Stargazers over time [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) + +[git.io/python-tools](https://git.io/python-tools) From 2c94c24a611b78b990d237165fd7e1937d9defa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:49:27 +0100 Subject: [PATCH 0730/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d67dcede2..7294e61f1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d67dcede2bbff47072b655ee641357723d91cc0f +Subproject commit 7294e61f1af8b2cf0a60b670136635b6d417fd5d From 1748e1da2fa62a36f8a65b2454aaf7b37bff1ef6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:49:27 +0100 Subject: [PATCH 0731/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0d1827a52..175ffba88 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0d1827a52e68a4174e8c1059971e884e2c14cb67 +Subproject commit 175ffba88fc740f0dbaf211d2675770affe5e382 From 9aecf1066d1be7186636318927a4c62feefdd4c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:25:51 +0100 Subject: [PATCH 0732/2295] updated schedule to 7am daily --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos6.yaml.disabled | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/python2.yaml | 2 +- .github/workflows/python3.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index b4ea64cf0..6d7345ccb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index b9d1305a4..db677d27e 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 13ff19170..1d98a2d0b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled index 6ec523536..dbbadfee1 100644 --- a/.github/workflows/centos6.yaml.disabled +++ b/.github/workflows/centos6.yaml.disabled @@ -24,7 +24,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 7ebef0745..0694d60a1 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3868ce406..2751546a5 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 9969fb0a1..635668ca1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 7980bae04..9fc388826 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index f0ec8860c..96673f75d 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 640d28a79..622715687 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index a7acf16cc..ca5afbfa5 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index e03e57161..0cfedafc6 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 1f7100849..e659a673d 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 0c1358164..87bbf2065 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index a16b0731f..22dfce874 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/python2.yaml b/.github/workflows/python2.yaml index e9489be32..c13d47aa4 100644 --- a/.github/workflows/python2.yaml +++ b/.github/workflows/python2.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/python3.yaml b/.github/workflows/python3.yaml index 16932c744..b6304c6e1 100644 --- a/.github/workflows/python3.yaml +++ b/.github/workflows/python3.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 840e5553a..313177077 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index f04e0bfda..29bb85bc9 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index c122a9cab..0795ff15c 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index ff17f01cd..d607fe25c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: From 9b2b4f86642726503e954810c8ac8f0bc2f895ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:26:00 +0100 Subject: [PATCH 0733/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7294e61f1..31cb96f72 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7294e61f1af8b2cf0a60b670136635b6d417fd5d +Subproject commit 31cb96f729fe0f83d6f5b6845a09d98b8d268ffa From dd152672d35ffb7bffbd0487433732d5ce8f85f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:26:00 +0100 Subject: [PATCH 0734/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 175ffba88..b99613841 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 175ffba88fc740f0dbaf211d2675770affe5e382 +Subproject commit b99613841ba4e867bd0eaa565efdfd45230a77e5 From 459d87ad391ed11dcc2b85a7db1b41e02c7f726a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:03:58 +0100 Subject: [PATCH 0735/2295] updated .appveyor.yml --- .appveyor.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index a97f97008..9d75f5a37 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -17,6 +17,14 @@ image: Ubuntu +# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ +environment: + APPVEYOR_SSH_KEY: AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== + +# enable SSH session accessible via my public key +init: + - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + install: # workaround for: # Some packages could not be installed. This may mean that you have From 1d979b780b19068ed2a72cc0e1e300c722271df5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:32:31 +0100 Subject: [PATCH 0736/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4490e2f81..80d0921dc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) -[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) +[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) From 59639c3a6fb1cbd43af334028854a2861cb2f420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:41:59 +0100 Subject: [PATCH 0737/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80d0921dc..f1f3c9027 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) -[![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) From fc52b36cc3d399d7c7919f85dd8e30ee2a8285a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:43:48 +0100 Subject: [PATCH 0738/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 31cb96f72..e78ac3856 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 31cb96f729fe0f83d6f5b6845a09d98b8d268ffa +Subproject commit e78ac38568d3cb6702da5e58f543d5d531653738 From d91a578eec4bd5608923e752ec5c4a29ada898a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:43:48 +0100 Subject: [PATCH 0739/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b99613841..ae3b05c6d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b99613841ba4e867bd0eaa565efdfd45230a77e5 +Subproject commit ae3b05c6da98d8f086359b0d5b630c39e72f9845 From 67a5eca57863a64ca84b7cdd0f7cbaac6895115a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:42:46 +0100 Subject: [PATCH 0740/2295] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9d75f5a37..ddf4ecdf9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ image: Ubuntu # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ environment: - APPVEYOR_SSH_KEY: AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== + APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension # enable SSH session accessible via my public key init: From 5b93d80aa0b75687e92ca612b758f0db161cba7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:59:51 +0100 Subject: [PATCH 0741/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e78ac3856..8333adc0c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e78ac38568d3cb6702da5e58f543d5d531653738 +Subproject commit 8333adc0cf42adcc4dc890c62ceb05afd352ea4c From 5cfe538682abf54e615976632fa67d4fc717a379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:59:51 +0100 Subject: [PATCH 0742/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae3b05c6d..ac6ba0d8a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae3b05c6da98d8f086359b0d5b630c39e72f9845 +Subproject commit ac6ba0d8a46c5ae84dedca563eed06a23ff58bbe From d4e7d57f95d02fb27bdc177ffc66e916e21d6d99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 19:18:56 +0100 Subject: [PATCH 0743/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8333adc0c..57937e3f4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8333adc0cf42adcc4dc890c62ceb05afd352ea4c +Subproject commit 57937e3f4d20f139daeaa1236fa47d9a20243da6 From f7f1f6adb3ff5e50fe3ee2e481242973d22fcb55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 19:18:56 +0100 Subject: [PATCH 0744/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ac6ba0d8a..0f144150e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ac6ba0d8a46c5ae84dedca563eed06a23ff58bbe +Subproject commit 0f144150ef41f84b7ca95dd59d7e64136df0ccf3 From 8656c125b4413f075f9f2d8c5bfe926237ded15b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 13:57:39 +0100 Subject: [PATCH 0745/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 57937e3f4..7539c4267 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 57937e3f4d20f139daeaa1236fa47d9a20243da6 +Subproject commit 7539c426766c44bbfdff1480fd15a46689d6745d From 5a6a5908c298bc7b4be045a8b39a09a50590b14b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 13:57:39 +0100 Subject: [PATCH 0746/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f144150e..114c7a54d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f144150ef41f84b7ca95dd59d7e64136df0ccf3 +Subproject commit 114c7a54d217b5b5eb729ccb4e5d8fe3ec6ca266 From c7da2a0087f7b86f576eea0e18730d968e4eb783 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 21:17:54 +0100 Subject: [PATCH 0747/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 114c7a54d..57b868bf0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 114c7a54d217b5b5eb729ccb4e5d8fe3ec6ca266 +Subproject commit 57b868bf096cd8574660becff7b9006996f48ec7 From 2742843fe5a016f49559c68898de2ef1d65d014c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:16:25 +0100 Subject: [PATCH 0748/2295] updated test_find_active_server.sh --- tests/test_find_active_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 1bde9e0e6..03e8dee75 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -144,7 +144,7 @@ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_server.py --https local echo "testing https with url path and regex matching:" echo -run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex 'python-tools' +run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex '(?i)python-tools' # ============================================================================ # From 3d255727c176193fadb634ab87f72d4a3d2a661f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:16:40 +0100 Subject: [PATCH 0749/2295] changed cpu_count import to work with Python 3 --- find_active_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index edcf0fb49..dea660dbb 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -112,7 +112,8 @@ import subprocess import sys #from threading import Thread -from multiprocessing.pool import ThreadPool, cpu_count +from multiprocessing.pool import ThreadPool +from multiprocessing import cpu_count # prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque' #from collections import deque import Queue @@ -137,7 +138,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.8.5' class FindActiveServer(CLI): From f0407797c76f1b91d8cde1fda25a5b34a4fc7d56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:23:58 +0100 Subject: [PATCH 0750/2295] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c63e3e540..99d0ac0b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,7 +103,7 @@ pipeline { // stage('Deployment') { // steps { // echo 'Deploying...' -// echo 'Nothing to deploy' +// echo 'Nothing to deploy' // } // } From 3f5a2605bf5d40a2cc6c1aa311406b6eae242f79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:41:26 +0100 Subject: [PATCH 0751/2295] updated .appveyor.yml --- .appveyor.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index ddf4ecdf9..0bce84c3e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,13 @@ environment: APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension # enable SSH session accessible via my public key -init: +#init: +# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + +# more useful at end to let .appveyor's tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to check out code and do fixes manually +on_finish: + # set this in Settings -> Environment dynamically instead of here + #- sh: export APPVEYOR_SSH_BLOCK=true - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - install: From 3ded1883c6adf0c799f35a71ff141ead14850d6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:38:36 +0100 Subject: [PATCH 0752/2295] more Python 3 fixes --- find_active_server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index dea660dbb..f759527eb 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -116,9 +116,13 @@ from multiprocessing import cpu_count # prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque' #from collections import deque -import Queue import traceback from random import shuffle +# Python 2 Queue vs Python 3 queue module :-/ +if sys.version[0] == '2': + import Queue as queue +else: + import queue as queue try: import requests except ImportError: @@ -138,7 +142,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.5' +__version__ = '0.8.6' class FindActiveServer(CLI): @@ -157,7 +161,7 @@ def __init__(self): self.request_timeout = None self.default_num_threads = min(cpu_count() * 4, 100) self.num_threads = None - self.queue = Queue.Queue() + self.queue = queue.Queue() self.pool = None def add_options(self): From 8a0087bd2e0e43f761a69e77eb575881eec98596 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:55:51 +0100 Subject: [PATCH 0753/2295] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0bce84c3e..7934f0253 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -25,7 +25,7 @@ environment: #init: # - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - -# more useful at end to let .appveyor's tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to check out code and do fixes manually +# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session on_finish: # set this in Settings -> Environment dynamically instead of here #- sh: export APPVEYOR_SSH_BLOCK=true From be1f1f2698ae1a55d7bec083d92ea3fb8e22e266 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:56:50 +0100 Subject: [PATCH 0754/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7539c4267..a708b2b76 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7539c426766c44bbfdff1480fd15a46689d6745d +Subproject commit a708b2b769ec3ddafedde7dbe8d02a7899b5df90 From 8b4521483b5d59a8b4908a85b65b74e8858bc341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:56:50 +0100 Subject: [PATCH 0755/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 57b868bf0..85f863007 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 57b868bf096cd8574660becff7b9006996f48ec7 +Subproject commit 85f863007a5259e077875b660248c69035410b2d From 949ec04f975f3e75168a60373a1cf8fa70a9212a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:06:25 +0100 Subject: [PATCH 0756/2295] renamed .github/workflows/python2.yaml to .github/workflows/python2.6.yaml --- .github/workflows/{python2.yaml => python2.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.yaml => python2.6.yaml} (100%) diff --git a/.github/workflows/python2.yaml b/.github/workflows/python2.6.yaml similarity index 100% rename from .github/workflows/python2.yaml rename to .github/workflows/python2.6.yaml From 211f6e038a004f135036dc10038fc1cb705cf9d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:06:32 +0100 Subject: [PATCH 0757/2295] renamed .github/workflows/python3.yaml to .github/workflows/python3.6.yaml --- .github/workflows/{python3.yaml => python3.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.yaml => python3.6.yaml} (100%) diff --git a/.github/workflows/python3.yaml b/.github/workflows/python3.6.yaml similarity index 100% rename from .github/workflows/python3.yaml rename to .github/workflows/python3.6.yaml From 51195606367f1c619ed4eb28ae6cbad1e3d42500 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:10:44 +0100 Subject: [PATCH 0758/2295] renamed .github/workflows/python2.6.yaml to .github/workflows/python2.7.yaml --- .github/workflows/{python2.6.yaml => python2.7.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.6.yaml => python2.7.yaml} (100%) diff --git a/.github/workflows/python2.6.yaml b/.github/workflows/python2.7.yaml similarity index 100% rename from .github/workflows/python2.6.yaml rename to .github/workflows/python2.7.yaml From eeb2fe8b43465c5f975483e6638b8120f6f35e2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:30:39 +0100 Subject: [PATCH 0759/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a708b2b76..9731e35b3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a708b2b769ec3ddafedde7dbe8d02a7899b5df90 +Subproject commit 9731e35b3bfeaece3b9706e3b2ec6f232041e22b From b3cf3887a8de79bf9d7e9da96b625897dc3c5ff9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:30:39 +0100 Subject: [PATCH 0760/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85f863007..9d38d37af 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85f863007a5259e077875b660248c69035410b2d +Subproject commit 9d38d37afeb88a09095cd04f5abd3d9e1ae8230b From a3cec30cf60e9527c9a8a311bec887f8fbf66e57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:11 +0100 Subject: [PATCH 0761/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c13d47aa4..89e33295e 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -27,12 +27,12 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [2.7] steps: - uses: actions/checkout@v2 @@ -42,8 +42,12 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- - - name: build & test - run: make build test + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 9c325dc77e0d6d89b031b784942f80cd7328a18e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:18 +0100 Subject: [PATCH 0762/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b6304c6e1..704b99878 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -27,12 +27,12 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [3.6] steps: - uses: actions/checkout@v2 @@ -42,8 +42,12 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- - - name: build & test - run: make build test + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 9458bb27559a8f2e3ff1627cf86d7886e52a4240 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:35 +0100 Subject: [PATCH 0763/2295] updated mac.yaml --- .github/workflows/mac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 87bbf2065..eb7a8ce81 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -39,7 +39,9 @@ jobs: ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || echo + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From 7617192bbca7d49bae31f3025409cd1495083914 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:48 +0100 Subject: [PATCH 0764/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 22dfce874..bc8004be0 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -39,7 +39,9 @@ jobs: ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || echo + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From 21bff4b4cc1b6bdeeac7409c6f17ccba82840cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:04 +0100 Subject: [PATCH 0765/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 313177077..c96960027 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -37,7 +37,9 @@ jobs: key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From ef7274b6aa73537de967160683c5b1f8359bbe3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:20 +0100 Subject: [PATCH 0766/2295] added python3.5.yaml --- .github/workflows/python3.5.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.5.yaml diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml new file mode 100644 index 000000000..79856bbc1 --- /dev/null +++ b/.github/workflows/python3.5.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.5 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.5] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From d436e331446f40e7f9b08b157e77bc9e11df965b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:21 +0100 Subject: [PATCH 0767/2295] added python3.7.yaml --- .github/workflows/python3.7.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.7.yaml diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml new file mode 100644 index 000000000..538144130 --- /dev/null +++ b/.github/workflows/python3.7.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.7 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 018b8964d042e9b1d1915c851d91666b9f0a2df8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:23 +0100 Subject: [PATCH 0768/2295] added python3.8.yaml --- .github/workflows/python3.8.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.8.yaml diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml new file mode 100644 index 000000000..6f1c69d4f --- /dev/null +++ b/.github/workflows/python3.8.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.8 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From ee8f045828f596bc71946c3655be829aa8d38186 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:25 +0100 Subject: [PATCH 0769/2295] added pypy2.yaml --- .github/workflows/pypy2.yaml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/pypy2.yaml diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml new file mode 100644 index 000000000..28a8a6445 --- /dev/null +++ b/.github/workflows/pypy2.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI PyPy 2 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [pypy2] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 388d81dcbdf58e41088c368f69e08d81718c20a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:27 +0100 Subject: [PATCH 0770/2295] added pypy3.yaml --- .github/workflows/pypy3.yaml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/pypy3.yaml diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml new file mode 100644 index 000000000..d1e8bf732 --- /dev/null +++ b/.github/workflows/pypy3.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI PyPy 3 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [pypy3] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 5ed7efb85ca23e9b53cf6519376630f1460ee789 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:34:07 +0100 Subject: [PATCH 0771/2295] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f1f3c9027..7f479e65d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Hari Sekhon - DevOps Python Tools [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Mac 10.15](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac%2010.15/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac+10.15%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) @@ -69,7 +70,12 @@ Hari Sekhon - DevOps Python Tools [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine+3%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) +[![CI Python 3.5](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.5/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.5%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) +[![CI Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.7%22) +[![CI Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.8%22) +[![CI PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+PyPy+2%22) +[![CI PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+PyPy+3%22) ### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### From d4e01a434b09008d7dc7332dc1a876fc4fd274e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:35:00 +0100 Subject: [PATCH 0772/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9731e35b3..0ff08f425 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9731e35b3bfeaece3b9706e3b2ec6f232041e22b +Subproject commit 0ff08f4255cc8b1534ef552b0f618ebff77d4477 From 1cb07cc1af7d65671e9dbfdabfb241bdb1a932c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:35:00 +0100 Subject: [PATCH 0773/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9d38d37af..86f6a3304 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9d38d37afeb88a09095cd04f5abd3d9e1ae8230b +Subproject commit 86f6a33046412ced2bf905082f29415a0f13a92b From b115e12862b73a94cdd45d826dafcb075ad742c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:59:23 +0100 Subject: [PATCH 0774/2295] updated find_active_server.py --- find_active_server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index f759527eb..1a127cd54 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -120,11 +120,12 @@ from random import shuffle # Python 2 Queue vs Python 3 queue module :-/ if sys.version[0] == '2': - import Queue as queue + import Queue as queue # pylint: disable=import-error else: - import queue as queue + import queue as queue # pylint: disable=import-error try: - import requests + # false positive from pylint, queue is imported first + import requests # pylint: disable=wrong-import-order except ImportError: print(traceback.format_exc(), end='') sys.exit(4) From 92c9d920a4008f8fa05635c6f52a77786d6e08e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 19:42:26 +0100 Subject: [PATCH 0775/2295] updated headtail.py --- headtail.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headtail.py b/headtail.py index ff061445a..ce9a43e43 100755 --- a/headtail.py +++ b/headtail.py @@ -41,7 +41,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.1' +__version__ = '0.3.2' class HeadTail(CLI): @@ -60,6 +60,7 @@ def __init__(self): self.sep = '-' * 80 self.docsep = '=' * 80 self.quiet = False + self.timeout_default = None def add_options(self): #self.timeout_default = 300 From 78e2932cd8682ae016c9160e45f25c99128dd340 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 23:02:22 +0100 Subject: [PATCH 0776/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 86f6a3304..0a5f65eae 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 86f6a33046412ced2bf905082f29415a0f13a92b +Subproject commit 0a5f65eae070fb0111c38e340d6dadf1e04c8a7f From 83f4ae1be5c6af89b1577f3b321c0d801bc5fa57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:46:34 +0100 Subject: [PATCH 0777/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7f479e65d..32f290d82 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Hari Sekhon - DevOps Python Tools [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) @@ -59,6 +60,7 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) +[![CI Ubuntu GitHub](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%20GitHub/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+GitHub%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) From 761b0f6344d4be2e8cd617d936a589b0741398cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:46:40 +0100 Subject: [PATCH 0778/2295] updated buddy.yml --- buddy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/buddy.yml b/buddy.yml index 9b897a700..19a4c04ce 100644 --- a/buddy.yml +++ b/buddy.yml @@ -28,11 +28,11 @@ docker_image_name: "library/ubuntu" docker_image_tag: "18.04" execute_commands: - - "apt update &&" - - "apt install -qy make &&" - - "make init &&" - - "make ci test" + - "apt update &&" + - "apt install -qy git make &&" + - "make init &&" + - "make ci test" volume_mappings: - - "/:/buddy/devops-python-tools" + - "/:/buddy/devops-python-tools" shell: "BASH" trigger_condition: "ALWAYS" From 7f92d4a64d43fa99e2487ac01f90b6980d3f6f4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:04 +0100 Subject: [PATCH 0779/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0ff08f425..09e5ac956 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0ff08f4255cc8b1534ef552b0f618ebff77d4477 +Subproject commit 09e5ac956e613a4968fb50c2317f37857bd9e4f8 From ed84dacf4735a4a06eef7f92898f09e65231764d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:04 +0100 Subject: [PATCH 0780/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a5f65eae..ea78f8b3d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a5f65eae070fb0111c38e340d6dadf1e04c8a7f +Subproject commit ea78f8b3d7d4aaf5debfa92a59c2a281ae4b561a From 8a2fd8fb27ca4ac1dfcdcab617676c326223a4f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:28 +0100 Subject: [PATCH 0781/2295] renamed ubuntu.yaml to ubuntu_github.yaml --- .github/workflows/{ubuntu.yaml => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu.yaml => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu.yaml rename to .github/workflows/ubuntu_github.yaml From e4491bd6239b5ff9f453952fdb3828465a448fb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:39 +0100 Subject: [PATCH 0782/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index c96960027..bd0f01189 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -11,7 +11,7 @@ # https://www.linkedin.com/in/harisekhon # -name: CI Ubuntu +name: CI Ubuntu GitHub #env: # DEBUG: 1 From 2a91910942868a1b3b412e58e61cf27d39458e8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:48:00 +0100 Subject: [PATCH 0783/2295] added ubuntu.yaml --- .github/workflows/ubuntu.yaml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ubuntu.yaml diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml new file mode 100644 index 000000000..9e3506d6f --- /dev/null +++ b/.github/workflows/ubuntu.yaml @@ -0,0 +1,47 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ubuntu-latest + container: ubuntu:latest + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: apt-get update -qq && apt-get install -qy git make + - name: git clone + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive + - name: build + run: cd "/tmp/$repo" && make ci + - name: test + run: cd "/tmp/$repo" && make test From fdf294510f5f00352391185b2230c922a89f0db7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 22:44:53 +0100 Subject: [PATCH 0784/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09e5ac956..ba415985e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09e5ac956e613a4968fb50c2317f37857bd9e4f8 +Subproject commit ba415985e088004b4617353c52a4246bf1e81d7b From 32c2a168241debafda7804b3bed8f9abb26b646b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 22:44:53 +0100 Subject: [PATCH 0785/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ea78f8b3d..401aa8728 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ea78f8b3d7d4aaf5debfa92a59c2a281ae4b561a +Subproject commit 401aa872810372013ee48435920c5c9f10c04220 From 1c49f975c1af79b9c65941b3f0ca79246b01b930 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 12 Apr 2020 21:20:57 +0100 Subject: [PATCH 0786/2295] added semaphore.yml --- .semaphore/semaphore.yml | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .semaphore/semaphore.yml diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000..fe6a604ee --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,71 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ + +version: v1.0 +name: devops-python-tools +agent: + machine: + type: e1-standard-2 + os_image: ubuntu1804 +execution_time_limit: + hours: 3 +blocks: + - name: Linux build + run: + when: "branch = 'master'" + #execution_time_limit: + # hours: 2 + task: + prologue: + commands: + - cache restore + jobs: + - name: install git & make + commands: + - sudo apt update -qq + - sudo apt install -qy git make + - name: build + commands: + - checkout + - make init + - make ci + - make test + epilogue: + commands: + - cache store + - name: Mac build + run: + when: "branch = 'masterX'" + task: + agent: + machine: + type: a1-standard-4 + os_image: macos-mojave-xcode10 + #os_image: macos-mojave-xcode11 + prologue: + commands: + - cache restore + jobs: + - name: build + commands: + - checkout + - make init + - make ci + - make test + epilogue: + commands: + - cache store From 8a6f7bd6af2b194c574217deb52a98367b807fb7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 12 Apr 2020 21:25:17 +0100 Subject: [PATCH 0787/2295] updated Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 4eb3ff5e1..7fa953349 100755 --- a/Makefile +++ b/Makefile @@ -62,7 +62,12 @@ build: init @# executing in sh where type is not available @#type -P python which python || : + which python2 || : + which python3 || : python -V || : + which pip || : + which pip2 || : + which pip3 || : pip -V || : if [ -z "$(CPANM)" ]; then make; exit $$?; fi From 501520a0308eb4ad322dd4ad310bc7795b3ec45b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Apr 2020 13:38:54 +0100 Subject: [PATCH 0788/2295] updated Makefile --- Makefile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 7fa953349..e241aabb7 100755 --- a/Makefile +++ b/Makefile @@ -61,14 +61,12 @@ build: init @# executing in sh where type is not available @#type -P python - which python || : - which python2 || : - which python3 || : - python -V || : - which pip || : - which pip2 || : - which pip3 || : - pip -V || : + which python && python -V || : + which python2 && python2 -V || : + which python3 && python3 -V || : + which pip && pip -V || : + which pip2 && pip2 -V || : + which pip3 && pip3 -V || : if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python From 32dd12db959e2e754c24f30ac0b505e3c68ea9d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Apr 2020 15:44:52 +0100 Subject: [PATCH 0789/2295] updated Makefile --- Makefile | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index e241aabb7..3376063e7 100755 --- a/Makefile +++ b/Makefile @@ -53,39 +53,25 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: init +build: init python-version @echo ========================= @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary - @# executing in sh where type is not available - @#type -P python - which python && python -V || : - which python2 && python2 -V || : - which python3 && python3 -V || : - which pip && pip -V || : - which pip2 && pip2 -V || : - which pip3 && pip3 -V || : - if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi - $(MAKE) python - # executing in sh where type is not available - #type -P python - which python - python -V - pip -V + $(MAKE) python .PHONY: init init: git submodule update --init --recursive .PHONY: python -python: +python: python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From 3be70ea4f98ebce70c168432c0ac127fd1149b9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:58:32 +0100 Subject: [PATCH 0790/2295] updated .semaphore --- .semaphore/semaphore.yml | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fe6a604ee..26ac24a3d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -30,17 +30,20 @@ blocks: #execution_time_limit: # hours: 2 task: + env_vars: + # to match /usr/local/bin/pip version from $PATH + - name: PYTHON + value: python3.8 prologue: commands: - cache restore + # each job is separate and could be run on a separate machine so all steps must be together jobs: - - name: install git & make - commands: - - sudo apt update -qq - - sudo apt install -qy git make - name: build commands: - checkout + - sudo apt update -qq + - sudo apt install -qy git make - make init - make ci - make test @@ -49,8 +52,17 @@ blocks: - cache store - name: Mac build run: - when: "branch = 'masterX'" + when: "branch = 'master'" task: + # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) + env_vars: + # to match /usr/local/bin/pip version from $PATH + - name: PYTHON + value: python3 + # must be quoted to force string, otherwise pipeline fails to run with this parsing error: + # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] + #- name: DEBUG + # value: "1" agent: machine: type: a1-standard-4 @@ -59,6 +71,23 @@ blocks: prologue: commands: - cache restore + # fix for: + # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available. + - brew install openssl + - brew reinstall python + - brew reinstall wget + # avoid Mac SSL errors: + # + # ERROR: Loading command: install (LoadError) + # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib + # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # ERROR: While executing gem ... (NoMethodError) + # undefined method `invoke_with_build_args' for nil:NilClass# + # + - rbenv global system + # also considered this: + # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done jobs: - name: build commands: From 289b48b51761aa3fb56c8f241c207a83a7484dbb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:59:34 +0100 Subject: [PATCH 0791/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ba415985e..e9323e43b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ba415985e088004b4617353c52a4246bf1e81d7b +Subproject commit e9323e43bc5e8f39d821be5fbca3522a84410536 From a2a316ebebbfe10904191bdc89f4bdc165a2719e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:59:34 +0100 Subject: [PATCH 0792/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 401aa8728..a1f3c1116 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 401aa872810372013ee48435920c5c9f10c04220 +Subproject commit a1f3c1116c613e32e7a85df01bf819b4ecc16dbc From 565c7d92d0fff4ad112018025af87f8a328abbee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:10:56 +0100 Subject: [PATCH 0793/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a1f3c1116..9e5deb8c3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a1f3c1116c613e32e7a85df01bf819b4ecc16dbc +Subproject commit 9e5deb8c32579f1e1ce7e11475a49e8bd9e33d4b From eec13571c5fd2078a6d6eb6bcf4fb746bddc5e22 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:11:19 +0100 Subject: [PATCH 0794/2295] updated pypy2.yaml pypy3.yaml --- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 28a8a6445..c71f2f9e5 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -32,7 +32,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [pypy2] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index d1e8bf732..ad306147d 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -32,7 +32,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [pypy3] steps: - uses: actions/checkout@v2 From 9baeeafdbb177819cc7a2fc81df8169dd0029b9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:12:38 +0100 Subject: [PATCH 0795/2295] updated Makefile --- Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Makefile b/Makefile index 3376063e7..a6706c6ab 100755 --- a/Makefile +++ b/Makefile @@ -185,11 +185,6 @@ test: test-lib basic-test: test-lib bash-tools/check_all.sh -.PHONY: test2 -test2: - cd pylib && $(MAKE) test2 - tests/all.sh - .PHONY: install install: build @echo "No installation needed, just add '$(PWD)' to your \$$PATH" From 4da2fa60f66774497ea031e468a4af2ee041954f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:30:53 +0100 Subject: [PATCH 0796/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9e5deb8c3..865dd9fe6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9e5deb8c32579f1e1ce7e11475a49e8bd9e33d4b +Subproject commit 865dd9fe6296f5611115adc2ba11752a3e11f80b From 6aa4b5cd71872b09f7058018c90c43e3bc561811 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:30:56 +0100 Subject: [PATCH 0797/2295] updated .travis.yml --- .travis.yml | 63 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ef2d390c..8cd9b2f40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,24 +20,61 @@ version: ~> 1.0 # - linux # - osx +python: + - "2.7" + #- "3.4" supported by pip as of March 2019 + - "3.5" + - "3.6" + - "3.7" + - "3.8" + - "pypy" # currently Python 2.7.13, PyPy 7.1.1 + - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 + matrix: + fast_finish: true include: + # numpy has gone 2.7+ only now, so had to drop Python 2.6 support - os: linux language: python - python: - # - "2.6" - - "2.7" - # MySQL in lib doesn't build from pip in Python 3 - # - "3.2" - # - "3.3" - # - "3.4" - # - "3.5" - # python-krbV fails to compile on PyPy - # - "pypy" - # - "pypy3" - # workaround is to use generic and install to system python + python: "2.7" + - os: osx - language: generic + language: generic # workaround since Mac doesn't have Python support yet, so install to system Python + + - os: linux + language: python + python: "3.5" + + - os: linux + language: python + python: "3.6" + + - os: linux + language: python + python: "3.7" + + - os: linux + language: python + python: "3.8" + + # python-krbV fails to compile on PyPy + - os: linux + language: python + python: "pypy" + + - os: linux + language: python + python: "pypy3" + + +# allow_failures: +# - python: "3.4" +# - python: "3.5" +# - python: "3.6" +# - python: "3.7" +# - python: "3.8" +# - python: "pypy" +# - python: "pypy3" dist: trusty From 792243c5c1d82c3ca4419477424b9f323ac92255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:32:13 +0100 Subject: [PATCH 0798/2295] updated .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8cd9b2f40..7ca66678f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: # - python: "pypy" # - python: "pypy3" -dist: trusty +#dist: trusty sudo: required From 032ecb2ffd12585131b113d6b0f06e7d1d489846 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 18:59:42 +0100 Subject: [PATCH 0799/2295] updated .travis.yml --- .travis.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ca66678f..cf0870f49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,10 +57,15 @@ matrix: language: python python: "3.8" - # python-krbV fails to compile on PyPy - - os: linux - language: python - python: "pypy" + # python-krbV fails to compile on PyPy + # + # psutil doesn't build: + # + # RuntimeError: broken / incompatible Python implementation, see: https://github.com/giampaolo/psutil/issues/1659 + # + #- os: linux + # language: python + # python: "pypy" - os: linux language: python From 2774c67202b883d59dd2d942aa458b037616f703 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 19:15:59 +0100 Subject: [PATCH 0800/2295] updated buddy.yml --- buddy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buddy.yml b/buddy.yml index 19a4c04ce..8257e4b8d 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,11 +27,12 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" + setup_commands: + - apt update + - apt install -qy git make execute_commands: - - "apt update &&" - - "apt install -qy git make &&" - - "make init &&" - - "make ci test" + - make init + - make ci test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From 0a5cbab7d19a2875b20e3bcba1563f694bbfb7c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 19:16:44 +0100 Subject: [PATCH 0801/2295] updated .appveyor.yml --- .appveyor.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 7934f0253..65331b992 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -29,6 +29,15 @@ environment: on_finish: # set this in Settings -> Environment dynamically instead of here #- sh: export APPVEYOR_SSH_BLOCK=true + # + # workaround for https://github.com/appveyor/ci/issues/3373 + # and https://github.com/appveyor/ci/issues/3384 + # + # has since been added to AppVeyor's own scripts: + # + # https://github.com/appveyor/ci/pull/3385 + # + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - install: From 9aa2a1113a4cceefe02f543cf1fcbfa71c6fbba7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Apr 2020 14:01:36 +0100 Subject: [PATCH 0802/2295] updated Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a6706c6ab..be45f98a7 100755 --- a/Makefile +++ b/Makefile @@ -21,15 +21,15 @@ # # Alpine: # -# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # # Debian / Ubuntu: # -# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # # RHEL / CentOS: # -# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # =================== From 61f627320ebea997558c603f4393f914fd86cd53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Apr 2020 14:18:09 +0100 Subject: [PATCH 0803/2295] updated welcome.py --- welcome.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/welcome.py b/welcome.py index bde562ac9..60d6533af 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.3' +__version__ = '2.0.4' class Welcome(CLI): @@ -59,7 +59,7 @@ def __init__(self): self.timeout_default = 20 @staticmethod - def case_user(user): + def titlecase_user(user): if user == 'root': user = user.upper() elif len(user) < 4 or re.search(r'\d', user): @@ -76,7 +76,7 @@ def construct_msg(self): # print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user) print("invalid user '%s' returned by getpass.getuser(), failed regex validation" % user) sys.exit(ERRORS['CRITICAL']) - user = self.case_user(user) + user = self.titlecase_user(user) msg = 'Welcome %s - ' % user last = '' if which("last"): From 84cc517484fccc3aca372c41364bac9a7470fea7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:34:36 +0100 Subject: [PATCH 0804/2295] updated .appveyor.yml --- .appveyor.yml | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e367a94c7..65331b992 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -17,7 +17,48 @@ image: Ubuntu +# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ +environment: + APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension + +# enable SSH session accessible via my public key +#init: +# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + +# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session +on_finish: + # set this in Settings -> Environment dynamically instead of here + #- sh: export APPVEYOR_SSH_BLOCK=true + # + # workaround for https://github.com/appveyor/ci/issues/3373 + # and https://github.com/appveyor/ci/issues/3384 + # + # has since been added to AppVeyor's own scripts: + # + # https://github.com/appveyor/ci/pull/3385 + # + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - + - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + install: +# workaround for: +# Some packages could not be installed. This may mean that you have +# requested an impossible situation or if you are using the unstable +# distribution that some required packages have not yet been created +# or been moved out of Incoming. +# The following information may help to resolve the situation: +# +# The following packages have unmet dependencies: +# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed +# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. +# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# make[2]: *** [apt-packages] Error 123 +# make[2]: Leaving directory '/home/appveyor/projects/pylib' +# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed +# +# adding "|| :" to the end of these commands causes them to be silently ignored! +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list +- sudo apt purge -qy --allow-change-held-packages mssql-server - sudo apt update -qq - sudo apt install -qy git make - make From c76907a8c59c9cb8691f4082116bc5cbce40db3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:35:04 +0100 Subject: [PATCH 0805/2295] updated config.yml --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7217d95a7..114368b87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,10 +19,20 @@ version: 2.1 jobs: build: + # technically a docker image is a better choice than machine + # but we want to introduce some native environment variation + # between build systems in order to test the repo's build automation is robust machine: - #image: ubuntu-1604:201903-01 image: default + #image: ubuntu-1604:201903-01 + # set to an actual docker image when running locally using circle_ci_job.sh + # docker image must have git installed to do the checkout + # so using harisekhon/dev:ubuntu instead of base ubuntu image + #image: harisekhon/dev:ubuntu steps: + # to allow docker networking to work + - run: sudo sysctl net.ipv4.ip_forward=1 + - run: sudo service docker restart - checkout - run: make init - run: make From 36392354a84e097e04f65fd794dc993e4dbad684 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:50:22 +0100 Subject: [PATCH 0806/2295] updated buddy.yml --- buddy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buddy.yml b/buddy.yml index 19a4c04ce..8257e4b8d 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,11 +27,12 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" + setup_commands: + - apt update + - apt install -qy git make execute_commands: - - "apt update &&" - - "apt install -qy git make &&" - - "make init &&" - - "make ci test" + - make init + - make ci test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From ada7e496c047903b272b3b918b11b7cca16d39d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:50:28 +0100 Subject: [PATCH 0807/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 26ac24a3d..f622c07a1 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -66,8 +66,7 @@ blocks: agent: machine: type: a1-standard-4 - os_image: macos-mojave-xcode10 - #os_image: macos-mojave-xcode11 + os_image: macos-xcode11 prologue: commands: - cache restore From 125fbc61716c7201dc0b518e14ead22fa2b7ea00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:51:12 +0100 Subject: [PATCH 0808/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e9323e43b..bf60f35d3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e9323e43bc5e8f39d821be5fbca3522a84410536 +Subproject commit bf60f35d33bf2aa38f18e7fb09d6f9a2125a7400 From 8084bfaa3b6d81f19be42ca0def6080b37ca2b4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:51:12 +0100 Subject: [PATCH 0809/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 865dd9fe6..ae53903b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 865dd9fe6296f5611115adc2ba11752a3e11f80b +Subproject commit ae53903b5f8a998084207b746d4fe4c956ef57b4 From 271af24995e6bfdeafc6f7ae565c570407f79d47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 17:58:55 +0100 Subject: [PATCH 0810/2295] added ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ubuntu_20.04.yaml diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml new file mode 100644 index 000000000..5f38e935c --- /dev/null +++ b/.github/workflows/ubuntu_20.04.yaml @@ -0,0 +1,47 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 20.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ubuntu-latest + container: ubuntu:20.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: apt-get update -qq && apt-get install -qy git make + - name: git clone + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive + - name: build + run: cd "/tmp/$repo" && make ci + - name: test + run: cd "/tmp/$repo" && make test From 7879f313019aef4e2c7b15082ee142615fce2442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 19:54:53 +0100 Subject: [PATCH 0811/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 32f290d82..f5893fc5f 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) +[![CI Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+20.04%22) [![CI Ubuntu GitHub](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%20GitHub/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+GitHub%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) From b364e74c6496b3265857382cfc350cec67e7816c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:16:57 +0100 Subject: [PATCH 0812/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bf60f35d3..90120262f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bf60f35d33bf2aa38f18e7fb09d6f9a2125a7400 +Subproject commit 90120262ff16a7c25a046d43f823335c5a442ded From d0d546213f1fc3d68098a72f92afae40202f1e4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:16:57 +0100 Subject: [PATCH 0813/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae53903b5..918826210 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae53903b5f8a998084207b746d4fe4c956ef57b4 +Subproject commit 918826210859758647ab6dc02d8a6af069e6e760 From b52989e0273123646add41b42d8da97e731e16cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:37:15 +0100 Subject: [PATCH 0814/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 90120262f..81fa2ce69 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 90120262ff16a7c25a046d43f823335c5a442ded +Subproject commit 81fa2ce6926642ebd671e74b3bc0b4a35c4208f4 From a64154e26a947753c438037c00a958c957a41589 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:37:15 +0100 Subject: [PATCH 0815/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 918826210..d866efd80 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 918826210859758647ab6dc02d8a6af069e6e760 +Subproject commit d866efd806a74b97b687902796d26f688be87a54 From 740399c7be3652682429b687f324ccc90df893b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:45:31 +0100 Subject: [PATCH 0816/2295] updated Makefile --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index be45f98a7..f13ff903a 100755 --- a/Makefile +++ b/Makefile @@ -53,11 +53,14 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: init python-version +build: init @echo ========================= @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + $(MAKE) python-version if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python @@ -71,7 +74,10 @@ init: git submodule update --init --recursive .PHONY: python -python: python-version +python: + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + $(MAKE) python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From 45e2ef423b28e38586233adaf87ef660c2b7b946 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:44 +0100 Subject: [PATCH 0817/2295] updated Makefile --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f13ff903a..d939369eb 100755 --- a/Makefile +++ b/Makefile @@ -58,9 +58,10 @@ build: init @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary + @echo # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - $(MAKE) python-version + @$(MAKE) python-version if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python @@ -77,7 +78,7 @@ init: python: # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - $(MAKE) python-version + @$(MAKE) python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From a6426138b4c8f4c864359afe02d1037363894f0b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:48 +0100 Subject: [PATCH 0818/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 81fa2ce69..fa89f29a2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 81fa2ce6926642ebd671e74b3bc0b4a35c4208f4 +Subproject commit fa89f29a29316ccfbd4fe33fc9207d4a9ea507dd From 76ab0836a999fa5f885e1d99f9437413d04e6e0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:49 +0100 Subject: [PATCH 0819/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d866efd80..7504ea948 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d866efd806a74b97b687902796d26f688be87a54 +Subproject commit 7504ea948018ad50374e1d0cae08d4baa44316cd From 9b704e57903b589d485139e5963852e80aa565f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 19:37:16 +0100 Subject: [PATCH 0820/2295] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 65331b992..16c3722e5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -38,7 +38,7 @@ on_finish: # https://github.com/appveyor/ci/pull/3385 # #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - - - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: # workaround for: From fb55c47d7f7b7cc112911b6f465f6784f2b92f94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 May 2020 14:57:14 +0100 Subject: [PATCH 0821/2295] updated .appveyor.yml --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 16c3722e5..7540e70a9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -59,6 +59,8 @@ install: # adding "|| :" to the end of these commands causes them to be silently ignored! - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list - sudo apt purge -qy --allow-change-held-packages mssql-server +# this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 +#- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages - sudo apt update -qq - sudo apt install -qy git make - make From 9488e1d593cdc1b2ca29693a72150411382c2c96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 May 2020 14:57:24 +0100 Subject: [PATCH 0822/2295] updated .semaphore --- .semaphore/semaphore.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index f622c07a1..4cfd2cb8e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -31,9 +31,11 @@ blocks: # hours: 2 task: env_vars: - # to match /usr/local/bin/pip version from $PATH + # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python - name: PYTHON - value: python3.8 + value: python3 + - name: PIP + value: pip3 prologue: commands: - cache restore From 914ac76360a2f70e4c2872b3b9c023a7c15b5274 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 May 2020 17:27:42 +0100 Subject: [PATCH 0823/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa89f29a2..e9371b29e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa89f29a29316ccfbd4fe33fc9207d4a9ea507dd +Subproject commit e9371b29e17e4ff8a091fbd1ce589c53ebea3679 From eddc03110a7d5f0ccf5dbd0da957703d97526ad3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 May 2020 17:27:42 +0100 Subject: [PATCH 0824/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7504ea948..545134be7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7504ea948018ad50374e1d0cae08d4baa44316cd +Subproject commit 545134be7d446b2e67b8cce0deba69ca0bbba2bd From 238ccca41414021e28b9b45f6e828744d283f2e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:08:06 +0100 Subject: [PATCH 0825/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5893fc5f..0f30f8aa9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) From 996cdf24b1258ff1e1a09576003f599468e5deaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:26:29 +0100 Subject: [PATCH 0826/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e9371b29e..f7318108b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e9371b29e17e4ff8a091fbd1ce589c53ebea3679 +Subproject commit f7318108b1ebfe6e013eed1d4266e1f5da46ff7a From 1c21f25a6adc62d3426049f9eaf9143d86e23161 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:26:29 +0100 Subject: [PATCH 0827/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545134be7..40fd56a17 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545134be7d446b2e67b8cce0deba69ca0bbba2bd +Subproject commit 40fd56a17e57b5ee616cb51a552ffe7b9f868aa4 From 77ac613cf4b2160c147a1846f30a26ff0390bae1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:46:14 +0100 Subject: [PATCH 0828/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f7318108b..c2f5c76bc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f7318108b1ebfe6e013eed1d4266e1f5da46ff7a +Subproject commit c2f5c76bca9aa6e4b6aba9398fbf0e861ed5bceb From f1fff3d5835d6e816fecf962bfb3b61f1a9baa43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:46:14 +0100 Subject: [PATCH 0829/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 40fd56a17..f6719de55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 40fd56a17e57b5ee616cb51a552ffe7b9f868aa4 +Subproject commit f6719de55889670fc458230b6b20a1f47bb6ecea From b0be3b6cc99beeccbe49d49aac4200dd23cc09fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:23:35 +0100 Subject: [PATCH 0830/2295] updated README.md --- README.md | 67 +++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 0f30f8aa9..95bea4b8f 100644 --- a/README.md +++ b/README.md @@ -158,19 +158,23 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days -- [Hadoop](http://hadoop.apache.org/) & NoSQL: - - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - see also ```validate_*.py``` further down for all these formats and more +- [Docker](https://www.docker.com/): + - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` + - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` + - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` +- [Spark](https://spark.apache.org/) & Data Format Converters: + - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter + - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter + - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter + - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter + - ```xml_to_json.py``` - XML to JSON converter + - ```json_to_xml.py``` - JSON to XML converter + - ```json_to_yaml.py``` - JSON to YAML converter + - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - see also ```validate_*.py``` further down for all these formats and more +- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints @@ -220,27 +224,22 @@ Environment variables are supported for convenience and also to hide credentials - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop - - ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark - - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) -- [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` +- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required + - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node + - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator + - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server + - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server + - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI From a29eaf12a7946e9b18d3ab13816821e730114d79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:25:29 +0100 Subject: [PATCH 0831/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95bea4b8f..289b61976 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ Environment variables are supported for convenience and also to hide credentials - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch). See further down for more details and sub-programs that simplify usage for the most common cluster technologies - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) From 17edfb63b0b3db4761fe083342ab070f55f28632 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:31:08 +0100 Subject: [PATCH 0832/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 289b61976..305b8b1d6 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Environment variables are supported for convenience and also to hide credentials - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch). See further down for more details and sub-programs that simplify usage for the most common cluster technologies + - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) From 84997dbc83f1a17a61225c3a762fd485069734a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 May 2020 19:23:51 +0100 Subject: [PATCH 0833/2295] set regex matching to apply only to file basename and not full path --- find_duplicate_files.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 00b45fe2e..7556c290c 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -33,7 +33,7 @@ 4. regex capture matching portion - specify a regex to match against the filenames with capture (brackets) and the captured portion will be compared among files. If no capture brackets are detected then will treat the entire regex as the capture. - Regex is case insensitive by default + Regex is case insensitive by default and applies only to the file's basename Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of @@ -81,7 +81,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.4' +__version__ = '0.6.0' class FindDuplicateFiles(CLI): @@ -165,7 +165,7 @@ def process_args(self): log_option('compare by name', self.compare_by_name) log_option('compare by size', self.compare_by_size) log_option('compare by checksum', self.compare_by_checksum) - log_option('compare by regex', True if self.regex else False) + log_option('compare by regex', bool(self.regex)) return args @staticmethod @@ -297,6 +297,7 @@ def check_path(self, path): def is_file_dup(self, filepath): log.debug("checking file path '%s'", filepath) + # pylint: disable=no-else-return if os.path.islink(filepath): log.debug("ignoring symlink '%s'", filepath) return False @@ -340,8 +341,7 @@ def is_file_dup_by_name(self, filepath): self.dups_by_name[basename].add(self.files[basename]) self.dups_by_name[basename].add(filepath) return True - else: - self.files[basename] = filepath + self.files[basename] = filepath return False def is_file_dup_by_size(self, filepath): @@ -397,7 +397,9 @@ def is_file_dup_by_hash(self, filepath): return False def is_file_dup_by_regex(self, filepath): - match = re.search(self.regex, filepath) + #match = re.search(self.regex, filepath) + basename = os.path.basename(filepath) + match = re.search(self.regex, basename) if match: log.debug("regex matched file '%s'", filepath) if match.groups(): @@ -407,8 +409,7 @@ def is_file_dup_by_regex(self, filepath): self.dups_by_regex[capture].add(self.regex_captures[capture]) self.dups_by_regex[capture].add(filepath) return True - else: - self.regex_captures[capture] = filepath + self.regex_captures[capture] = filepath else: log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?') return False From 122b0fea4309b4e5937fbee157e464b8a0227ed7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 May 2020 09:57:24 +0100 Subject: [PATCH 0834/2295] defaulted to capturing entire regex when no capture group is given --- find_duplicate_files.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 7556c290c..b1f35742e 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -81,7 +81,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class FindDuplicateFiles(CLI): @@ -404,14 +404,14 @@ def is_file_dup_by_regex(self, filepath): log.debug("regex matched file '%s'", filepath) if match.groups(): capture = match.group(1) - if capture in self.regex_captures: - self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) - self.dups_by_regex[capture].add(self.regex_captures[capture]) - self.dups_by_regex[capture].add(filepath) - return True - self.regex_captures[capture] = filepath else: - log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?') + capture = match.group(0) + if capture in self.regex_captures: + self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) + self.dups_by_regex[capture].add(self.regex_captures[capture]) + self.dups_by_regex[capture].add(filepath) + return True + self.regex_captures[capture] = filepath return False From f1ccc9670bb9b6f3d566a7b8ddf445a7064356bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 May 2020 11:49:42 +0100 Subject: [PATCH 0835/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c2f5c76bc..3f3c2f918 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c2f5c76bca9aa6e4b6aba9398fbf0e861ed5bceb +Subproject commit 3f3c2f918b242e32a112a182ac461ab66331bab0 From f40eeceffd4a6e698ab6d71401be24eba507b9bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 May 2020 11:49:44 +0100 Subject: [PATCH 0836/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f6719de55..93c911150 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f6719de55889670fc458230b6b20a1f47bb6ecea +Subproject commit 93c9111508dbbb756f99c8062c095580cf6bc203 From d64e76f6ec5142087f5bf47ac5031be38b106dab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:28:57 +0100 Subject: [PATCH 0837/2295] added barclaycard_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..5b6ed56f8 --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +#. "$srcdir/lib/utils.sh" + +# statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv +STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +converter="$srcdir/../crunch_accounting_csv_statement_converter.py" + +for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + latest_crunch_statement="$crunch_statement" + fi +done + +get_starting_balance(){ + local crunch_statement="$1" + if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then + echo "invalid statement passed to get_starting_balance, must be *_crunch.csv" >&2 + exit 1 + fi + tail -n 1 "$crunch_statement" | awk -F, '{print $4}' +} + +if [ -n "${latest_crunch_statement:-}" ]; then + echo "latest crunch statement is $latest_crunch_statement" + starting_balance="$(get_starting_balance "$latest_crunch_statement")" +else + starting_balance="${STARTING_BALANCE:-}" + if [ -n "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + exit 1 + fi +fi + +echo "starting balance: $starting_balance" + +passed_latest_statement=0 + +for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + continue + fi + if [ $passed_latest_statement -lt 1 ]; then + continue + fi + "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + starting_balance="$(get_starting_balance "$crunch_statement")" +done From a79b43a99638617883d7cf59e0364e7f638a3945 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:39:55 +0100 Subject: [PATCH 0838/2295] updated barclaycard_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 52 ++----------------- 1 file changed, 3 insertions(+), 49 deletions(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh index 5b6ed56f8..cbf3568e3 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -18,55 +18,9 @@ set -euo pipefail srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090 -#. "$srcdir/lib/utils.sh" +. "$srcdir/lib.sh" # statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv -STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" +export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" -converter="$srcdir/../crunch_accounting_csv_statement_converter.py" - -for statement in $STATEMENT_GLOB; do - crunch_statement="${statement%.csv}_crunch.csv" - if [ -f "$crunch_statement" ]; then - latest_crunch_statement="$crunch_statement" - fi -done - -get_starting_balance(){ - local crunch_statement="$1" - if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then - echo "invalid statement passed to get_starting_balance, must be *_crunch.csv" >&2 - exit 1 - fi - tail -n 1 "$crunch_statement" | awk -F, '{print $4}' -} - -if [ -n "${latest_crunch_statement:-}" ]; then - echo "latest crunch statement is $latest_crunch_statement" - starting_balance="$(get_starting_balance "$latest_crunch_statement")" -else - starting_balance="${STARTING_BALANCE:-}" - if [ -n "$starting_balance" ]; then - echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 - exit 1 - fi -fi - -echo "starting balance: $starting_balance" - -passed_latest_statement=0 - -for statement in $STATEMENT_GLOB; do - crunch_statement="${statement%.csv}_crunch.csv" - if [ -f "$crunch_statement" ]; then - if [ "$crunch_statement" = "$latest_crunch_statement" ]; then - passed_latest_statement=1 - fi - continue - fi - if [ $passed_latest_statement -lt 1 ]; then - continue - fi - "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" - starting_balance="$(get_starting_balance "$crunch_statement")" -done +generate_crunch_statements From 26afc75e240ff1d126e82e34f65cf7ca72924aa4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:39:59 +0100 Subject: [PATCH 0839/2295] added lib.sh --- .../lib.sh | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/lib.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh new file mode 100755 index 000000000..f6079e7fb --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +#. "$srcdir/lib/utils.sh" + +converter="$srcdir/../crunch_accounting_csv_statement_converter.py" + +get_latest_crunch_statement(){ + local latest_crunch_statement + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + latest_crunch_statement="$crunch_statement" + fi + done + if [ -n "$latest_crunch_statement" ]; then + echo "$latest_crunch_statement" + fi +} + +get_final_balance_from_statement(){ + local crunch_statement="$1" + if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then + echo "invalid statement passed to get_final_balance_from_statement(), must be *_crunch.csv" >&2 + exit 1 + fi + tail -n 1 "$crunch_statement" | awk -F, '{print $4}' +} + +get_starting_balance(){ + local starting_balance + if [ -n "${latest_crunch_statement:-}" ]; then + echo "latest crunch statement is $latest_crunch_statement" >&2 + starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")" + else + starting_balance="${STARTING_BALANCE:-}" + if [ -n "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + exit 1 + fi + fi + echo "starting balance: $starting_balance" >&2 + echo "$starting_balance" +} + +generate_crunch_statements(){ + local passed_latest_statement=0 + local latest_crunch_statement + local starting_balance + latest_crunch_statement="$(get_latest_crunch_statement)" + starting_balance="$(get_starting_balance "$latest_crunch_statement")" + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + continue + fi + if [ $passed_latest_statement -lt 1 ]; then + continue + fi + "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + starting_balance="$(get_final_balance_from_statement "$crunch_statement")" + done +} From bf1b85941ece805ed7b1dff66fd5a750b035a23d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:06:57 +0100 Subject: [PATCH 0840/2295] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 9755b4435..1bf4d9e75 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.0' +__version__ = '0.7.1' class CrunchAccountingCsvStatementConverter(CLI): @@ -107,6 +107,7 @@ def run(self): log.info("converted '%s' => '%s'", filename, target_filename) else: log.error("FAILED to convert filename '%s'", filename) + sys.exit(2) log.info('Final Balance: {}'.format(self.running_balance)) def convert(self, filename, target_filename): From 651f4f9f14d1a310aa841a8f0c554681e1ca7d07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:12:18 +0100 Subject: [PATCH 0841/2295] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 1bf4d9e75..ab51cdac1 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.1' +__version__ = '0.7.2' class CrunchAccountingCsvStatementConverter(CLI): @@ -168,13 +168,17 @@ def detect_columns(self, csvreader): # want Transaction Date and not Posted Date if 'Date' in value and not 'Posted' in value: positions['date'] = position - elif 'Merchant Name' in value: - positions['desc'] = position # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33 elif 'Amount' in value and not 'Original' in value: positions['amount'] = position elif 'Balance' in value: balance_position = position + # Barclaycard CSVs + elif 'Merchant Name' in value: + positions['desc'] = position + # Barclays CSVs + elif 'Memo' in value: + positions['desc'] = position for pos in positions: if positions[pos] is None: log.error('field %s not found', pos) From 7b9da2f721f4b379109cc9958ec109c88ea4f851 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:16:49 +0100 Subject: [PATCH 0842/2295] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index ab51cdac1..4517f97b7 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.7.3' class CrunchAccountingCsvStatementConverter(CLI): @@ -225,7 +225,7 @@ def validate_csvreader(csvreader, filename): # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation - if not isChars(field_list[0][0], 'A-Za-z0-9"'): + if field_list[0] != "" and not isChars(field_list[0][0], 'A-Za-z0-9"'): log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 From 9971187d8671b6f3a3956dbbafb4c0382d47fa7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:22:04 +0100 Subject: [PATCH 0843/2295] updated lib.sh --- .../lib.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh index f6079e7fb..d6c217880 100755 --- a/crunch_accounting_csv_statement_converter_scripts/lib.sh +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -30,7 +30,7 @@ get_latest_crunch_statement(){ latest_crunch_statement="$crunch_statement" fi done - if [ -n "$latest_crunch_statement" ]; then + if [ -n "${latest_crunch_statement:-}" ]; then echo "$latest_crunch_statement" fi } @@ -46,13 +46,15 @@ get_final_balance_from_statement(){ get_starting_balance(){ local starting_balance + local latest_crunch_statement="$1" if [ -n "${latest_crunch_statement:-}" ]; then echo "latest crunch statement is $latest_crunch_statement" >&2 starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")" else + echo "no latest crunch statement, getting starting balance from environment variable \$STARTING_BALANCE" >&2 starting_balance="${STARTING_BALANCE:-}" - if [ -n "$starting_balance" ]; then - echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + if [ -z "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$STARTING_BALANCE" >&2 exit 1 fi fi @@ -61,15 +63,20 @@ get_starting_balance(){ } generate_crunch_statements(){ + # only generate statements newer than the last generated one which provides the starting balance local passed_latest_statement=0 local latest_crunch_statement local starting_balance latest_crunch_statement="$(get_latest_crunch_statement)" + if [ -z "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi starting_balance="$(get_starting_balance "$latest_crunch_statement")" for statement in $STATEMENT_GLOB; do crunch_statement="${statement%.csv}_crunch.csv" if [ -f "$crunch_statement" ]; then - if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + if [ $passed_latest_statement = 0 ] && + [ "$crunch_statement" = "$latest_crunch_statement" ]; then passed_latest_statement=1 fi continue From 8ac9b8f0ebd46a5732f9a2490364065910d0467f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:23:48 +0100 Subject: [PATCH 0844/2295] added barclays_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..f667216a6 --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +. "$srcdir/lib.sh" + +# statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv +export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +# Barclays CSV statements often have whitespace starting fields instead of blank or 'null' +# unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed +# UPDATE: no longer necessary, converter just ignores these blank fields now in the validation +#for statement in $STATEMENT_GLOB; do + #perl -pi -e 's/^\s+,/,/' "$statement" +#done + +generate_crunch_statements From 0a41818ffc74ba69fafd703e4f94b2be54a76732 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:34 +0100 Subject: [PATCH 0845/2295] updated lib.sh --- crunch_accounting_csv_statement_converter_scripts/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh index d6c217880..65a21926b 100755 --- a/crunch_accounting_csv_statement_converter_scripts/lib.sh +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -84,7 +84,7 @@ generate_crunch_statements(){ if [ $passed_latest_statement -lt 1 ]; then continue fi - "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + "$converter" "$@" --starting-balance "$starting_balance" "$statement" starting_balance="$(get_final_balance_from_statement "$crunch_statement")" done } From ca99e1ab3b47d5a98401a621e87d403c053a6998 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:38 +0100 Subject: [PATCH 0846/2295] updated barclaycard_crunch_accounting_csv_statement_converter.sh --- .../barclaycard_crunch_accounting_csv_statement_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh index cbf3568e3..708ab7a66 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -23,4 +23,4 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" -generate_crunch_statements +generate_crunch_statements --credit-card --reverse-order From cf9da7a9e888d9d2d42989c73f31089c64e2b8e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:42 +0100 Subject: [PATCH 0847/2295] updated barclays_crunch_accounting_csv_statement_converter.sh --- .../barclays_crunch_accounting_csv_statement_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh index f667216a6..704b9defd 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh @@ -30,4 +30,4 @@ export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:di #perl -pi -e 's/^\s+,/,/' "$statement" #done -generate_crunch_statements +generate_crunch_statements --reverse-order From 825738646eb9081de400db1800f7a5bd076e86cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:45:50 +0100 Subject: [PATCH 0848/2295] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 4517f97b7..102b84058 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -182,7 +182,7 @@ def detect_columns(self, csvreader): for pos in positions: if positions[pos] is None: log.error('field %s not found', pos) - return False + sys.exit(1) if balance_position is None and self.running_balance is None: self.usage('no balance column detected, please specify --starting-balance') return (positions, balance_position) From 248b6b1dbb11448e4cefd9836a34566729efad00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:55:11 +0100 Subject: [PATCH 0849/2295] updated buddy.yml --- buddy.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buddy.yml b/buddy.yml index 8257e4b8d..533d34910 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,10 +27,14 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" - setup_commands: + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: - apt update - apt install -qy git make - execute_commands: - make init - make ci test volume_mappings: From d5e5da0614337498cc9278ddffe6ad1768b2208e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:55:17 +0100 Subject: [PATCH 0850/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3f3c2f918..d8957934d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3f3c2f918b242e32a112a182ac461ab66331bab0 +Subproject commit d8957934d680bbf1fae688b7ee8cb543d1b65e0b From 2d5fd10c50ba1bc6c1fa7183c52925aaf7caba53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:56:26 +0100 Subject: [PATCH 0851/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93c911150..5e12d6ac8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93c9111508dbbb756f99c8062c095580cf6bc203 +Subproject commit 5e12d6ac8a60d4eb90932a3b97773809eb70a754 From 68a9bb621af1ce02b72ec3e93a57fc00c1f9564a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 14:43:38 +0100 Subject: [PATCH 0852/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d8957934d..e3048bd4b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d8957934d680bbf1fae688b7ee8cb543d1b65e0b +Subproject commit e3048bd4be961071db74cd302a565fdac06d8ce2 From fc073dd92ec097535c5882c4e59ca10217ff92ac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 14:43:38 +0100 Subject: [PATCH 0853/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5e12d6ac8..56bf21007 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5e12d6ac8a60d4eb90932a3b97773809eb70a754 +Subproject commit 56bf21007849feeb15f4cb056a52b55deebda604 From f1cde920455d2fa5695d4077488aedb3fd5d9c5c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 15:56:13 +0100 Subject: [PATCH 0854/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e3048bd4b..1152d2f83 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e3048bd4be961071db74cd302a565fdac06d8ce2 +Subproject commit 1152d2f83b34e2adbbfa278188eef5ee0f6bae55 From f76040271560c9cc03e5be3c760b4154066c7868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 15:56:13 +0100 Subject: [PATCH 0855/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 56bf21007..3933b12fe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 56bf21007849feeb15f4cb056a52b55deebda604 +Subproject commit 3933b12fef600a07959df49732714c614721bc12 From 24a0c3680c3925ff85a4a15b8caf26a8ba3d54fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 17:07:06 +0100 Subject: [PATCH 0856/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1152d2f83..408468520 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1152d2f83b34e2adbbfa278188eef5ee0f6bae55 +Subproject commit 408468520d273f36a827a1b9d62158a3d4db7ff4 From b1b7778c015cfcab01dae2a1e2cbc54fe0fa6351 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 17:07:06 +0100 Subject: [PATCH 0857/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3933b12fe..f87fd873b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3933b12fef600a07959df49732714c614721bc12 +Subproject commit f87fd873b84579696bfb90fb46ee66643d84ecfb From e43d1f07d81882776319e1c22dc7235d519123b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:08:25 +0100 Subject: [PATCH 0858/2295] updated .appveyor.yml --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7540e70a9..4488bbf71 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,8 +61,8 @@ install: - sudo apt purge -qy --allow-change-held-packages mssql-server # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- sudo apt update -qq -- sudo apt install -qy git make +- for x in `seq 10`; do sudo apt update -qq && break; sleep 60; done +- for x in `sed 10`; do sudo apt install -qy git make && break; sleep 60; done - make test_script: From 5546269378424a87b6bfdfbb81ca153b5e757a14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:10:40 +0100 Subject: [PATCH 0859/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 408468520..1dd707e62 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 408468520d273f36a827a1b9d62158a3d4db7ff4 +Subproject commit 1dd707e6228c9bda2bad2a684dfc3697237ed0b0 From e04d86e4450c4d313b66e2733d49494aaddaefda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:10:40 +0100 Subject: [PATCH 0860/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f87fd873b..9d3c31caf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f87fd873b84579696bfb90fb46ee66643d84ecfb +Subproject commit 9d3c31caf3d3efb90866ad225d3d469263d962a2 From 2d4c4819871e6ea8fc2b8e565ea610b1318cdbc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 23:46:21 +0100 Subject: [PATCH 0861/2295] updated test_anonymize.py --- tests/test_anonymize.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 6850e650b..4bd3aebc0 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -34,11 +34,11 @@ def run(): global dest # pylint: disable=global-statement src = {int(k) : v for k, v in src.items()} dest = {int(k) : v for k, v in dest.items()} - src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name - test_input = '\n'.join([src[key] for key in src_keys]) + src_keys = sorted(src) + test_input = '\n'.join([src[_] for _ in src_keys]) print('running anonymize tests using: {} {}'.format(anonymize, args)) - cmd = [anonymize] + [_ for _ in args.split()] + cmd = [anonymize] + args.split() process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) # encode as bytes for Python 3 :-/ test_input = str.encode(test_input, 'utf-8') @@ -46,7 +46,8 @@ def run(): index = 0 # convert bytes to string stdout = stdout.decode("utf-8") - for line in stdout.split('\n'): # pylint: disable=redefined-outer-name + # pylint: disable=redefined-outer-name + for line in stdout.split('\n'): key = src_keys[index] _input = src[key] expected = dest[key] From 72ca6a6ea61d3f4fdf937cb82d7eda00be9a15e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 4 Jun 2020 14:57:33 +0100 Subject: [PATCH 0862/2295] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index d46b6c796..10681eac0 100755 --- a/anonymize.py +++ b/anonymize.py @@ -81,6 +81,8 @@ ip_regex, \ subnet_mask_regex, \ user_regex + # used dynamically + # pylint: disable=unused-import from harisekhon.utils import \ domain_regex, \ email_regex, \ From 5179ca27cecf1d5b801a1ae37d1efda5f460f0c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 4 Jun 2020 15:01:25 +0100 Subject: [PATCH 0863/2295] updated anonymize.py --- anonymize.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/anonymize.py b/anonymize.py index 10681eac0..bf87520a6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -357,8 +357,8 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), - 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(\ - group='({}/)*{}'.format(user_regex, user_regex)), + 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format( + group='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -370,7 +370,8 @@ def __init__(self): 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex), - 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(user='({}/)*{}'.format(user_regex, user_regex)), + 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format( + user='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'password': r'([\.-]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -658,6 +659,7 @@ def process_options(self): self.anonymizations['ip'] = False else: for _ in self.anonymizations: + # pylint: disable=no-else-continue if _ in ('subnet_mask', 'mac', 'group'): continue elif _ == 'database': @@ -891,8 +893,7 @@ def anonymize(self, line): if line is None: if method: raise AssertionError('anonymize_{} returned None'.format(_)) - else: - raise AssertionError('anonymize_dynamic({}, line)'.format(_)) + raise AssertionError('anonymize_dynamic({}, line)'.format(_)) line += line_ending return line From 02d0ddd2fb16e6974f2dc80a0e84d635c4883222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 13:58:05 +0100 Subject: [PATCH 0864/2295] updated config.yml --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 114368b87..9b803fbad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,10 +30,12 @@ jobs: # so using harisekhon/dev:ubuntu instead of base ubuntu image #image: harisekhon/dev:ubuntu steps: - # to allow docker networking to work - - run: sudo sysctl net.ipv4.ip_forward=1 - - run: sudo service docker restart - checkout + - run: for x in `seq 10`; do sudo apt update -q && break; sleep 60; done + - run: for x in `seq 10`; do sudo apt install -qy git make && break; sleep 60; done - run: make init - run: make + # to allow docker networking to work + - run: sudo sysctl net.ipv4.ip_forward=1 + - run: sudo service docker restart - run: make test From 9b80a6578b62cff0d80b2ae5d31bf11a3fdcde69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 14:23:28 +0100 Subject: [PATCH 0865/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4cfd2cb8e..89c409ada 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -30,12 +30,12 @@ blocks: #execution_time_limit: # hours: 2 task: - env_vars: + #env_vars: # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python - - name: PYTHON - value: python3 - - name: PIP - value: pip3 + #- name: PYTHON + # value: python3 + #- name: PIP + # value: pip3 prologue: commands: - cache restore @@ -57,10 +57,10 @@ blocks: when: "branch = 'master'" task: # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) - env_vars: + #env_vars: # to match /usr/local/bin/pip version from $PATH - - name: PYTHON - value: python3 + #- name: PYTHON + # value: python3 # must be quoted to force string, otherwise pipeline fails to run with this parsing error: # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] #- name: DEBUG @@ -89,6 +89,9 @@ blocks: - rbenv global system # also considered this: # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done + # + # fix for python vs pip version mismatch + - ln -svf /usr/local/bin/python3 /usr/local/bin/python jobs: - name: build commands: From db9bee094377db3140add8230ee1231adb71e420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:14:00 +0100 Subject: [PATCH 0866/2295] added ci_bootstrap.sh --- setup/ci_bootstrap.sh | 83 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 setup/ci_bootstrap.sh diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh new file mode 100644 index 000000000..dc2e22b00 --- /dev/null +++ b/setup/ci_bootstrap.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding +# +# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems + +set -eu +[ -n "${DEBUG:-}" ] && set -x + +max_tries=10 +interval=60 # secs + +sudo="" +# EUID undefined in posix sh +#[ $EUID = 0 ] || sudo=sudo +[ "$(whoami)" = root ] || sudo=sudo + +retry(){ + # no local in posix sh + count=0 + while true; do + # no let or bare (()) in posix sh, must discard output rather than execute it + _=$((count+=1)) + printf "%s try %d: " "$(date '+%F %T')" "$count" + echo "$*" + "$@" && + break; + echo + if [ $count -ge $max_tries ]; then + echo "$count tries failed, aborting..." + exit 1 + fi + echo "sleeping for $interval secs before retrying" + sleep "$interval" + echo + done +} + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) + #retry "$srcdir/install_homebrew.sh" + if command -v brew 2>&1; then + retry $sudo brew update + fi +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type -P apk >/dev/null 2>&1; then + retry $sudo apk update + retry $sudo apk add --no-progress bash git make + elif type apt-get >/dev/null 2>&1; then + retry $sudo apt-get update -q + retry $sudo apt-get install -qy git make + elif type yum >/dev/null 2>&1; then + #retry $sudo yum makecache + retry $sudo yum install -qy git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +#retry make init + +# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build +# +#make From ecea5c522ee9cf34c7a681aaf39ed1bcb2de504f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:57 +0100 Subject: [PATCH 0867/2295] updated Jenkinsfile --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 99d0ac0b7..c9fbad7d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,8 +70,7 @@ pipeline { // sh 'apt install -qy make' // sh 'make init' sh """ - apt update -q && - apt install -qy make && + setup/ci_bootstrap.sh && make init """ } From b99d70ea2c325322177742d80dc7f7f45b0f1d4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 0868/2295] updated .appveyor.yml --- .appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4488bbf71..eb43a0250 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,8 +61,7 @@ install: - sudo apt purge -qy --allow-change-held-packages mssql-server # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- for x in `seq 10`; do sudo apt update -qq && break; sleep 60; done -- for x in `sed 10`; do sudo apt install -qy git make && break; sleep 60; done +- setup/ci_bootstrap.sh - make test_script: From bcc02a8ecfe4e00301cd29c5ef09ec4b8f3d745b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 0869/2295] updated pipeline.yml --- .buildkite/pipeline.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c8bfbe881..33f4e790a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -20,22 +20,9 @@ # - command: buildkite-agent pipeline upload steps: - - command: | - type make 2>/dev/null || - if type apk 2>/dev/null; then - apk add --no-cache --no-progress make - # apt is /usr/bin/apt - # Unable to locate an executable at "/Users/hari/.sdkman/candidates/java/current/bin/apt" (-1) - elif type apt-get 2>/dev/null; then - apt-get update -q && - apt-get install -qy make - elif type yum 2>/dev/null; then - rpm -q make || yum install -y make - elif type brew 2>/dev/null; then - brew install make - fi - label: install make - timeout: 20 # brew can take 10 mins just to do a brew update + - command: setup/ci_bootstrap.sh + label: ci bootstrap + timeout: 30 # brew can take 10 mins just to do a brew update - wait - command: make init label: init From bd18c97dadb091a4b6f74db39d1a54d29bc02b2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 0870/2295] updated config.yml --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b803fbad..69f7a6179 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,8 +31,7 @@ jobs: #image: harisekhon/dev:ubuntu steps: - checkout - - run: for x in `seq 10`; do sudo apt update -q && break; sleep 60; done - - run: for x in `seq 10`; do sudo apt install -qy git make && break; sleep 60; done + - run: setup/ci_bootstrap.sh - run: make init - run: make # to allow docker networking to work From 208abb3afeeaec7f3718b218d34d406edde5e830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 0871/2295] updated .cirrus.yml --- .cirrus.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 38752ca97..46de8ac49 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -13,8 +13,13 @@ # https://www.linkedin.com/in/harisekhon # +# https://cirrus-ci.org/guide/writing-tasks/ + container: image: ubuntu:18.04 task: - script: apt update -qq && apt install -qy git make && make init && make ci test + script: + - setup/ci_bootstrap.sh + - make init + - make ci test From 4705d70fa8f9edef67a15b300aeb69c9011ed65b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 0872/2295] updated .concourse.yml --- .concourse.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.concourse.yml b/.concourse.yml index d50f7ecf4..2abb4ac17 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -53,7 +53,6 @@ jobs: - -c - | cd code && - apt update -q && - apt install -qy git make && + setup/ci_bootstrap.sh && make init && make ci test From 61f5eab2c816c31f56e2c1430fd078f97a97ad6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 0873/2295] updated .drone.yml --- .drone.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 4dd6036f6..f57a8d178 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -23,8 +23,7 @@ steps: # environment: # DEBUG: 1 commands: - - apt update -qq - - apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From 931737931b814c27f8abd58b81e9d7856be61584 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 0874/2295] updated .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1613d1410..af2a6a22b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,6 @@ image: ubuntu:18.04 job: before_script: - - apt-get update -qq && apt-get install -yq git make + - setup/ci_bootstrap.sh script: - make init && make ci test From c32cbdd454aba2b62eb806082e7e461df56b9226 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 0875/2295] updated .gocd.yml --- .gocd.yml | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.gocd.yml b/.gocd.yml index ad1c3d693..54f7a3067 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -38,24 +38,30 @@ pipelines: type: success allow_only_on_success: false jobs: - apt-update: +# apt-update: +# timeout: 10 +# tasks: +# - exec: +# command: apt +# arguments: +# - update +# run_if: passed +# install-make: +# timeout: 10 +# tasks: +# - exec: +# command: apt +# arguments: +# - install +# - -qy +# - git +# - make +# run_if: passed + ci-bootstrap: timeout: 10 tasks: - exec: - command: apt - arguments: - - update - run_if: passed - install-make: - timeout: 10 - tasks: - - exec: - command: apt - arguments: - - install - - -qy - - git - - make + command: setup/ci_bootstrap.sh run_if: passed init: timeout: 10 From 5f7849f1c6dd115b92659c50dd85a3c3f3975f13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 0876/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 89c409ada..09b556190 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,8 +44,7 @@ blocks: - name: build commands: - checkout - - sudo apt update -qq - - sudo apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From 1e056326b0a04d15056a601ff1405a704e3633ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 0877/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cdf98c65a..0850b087f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,5 +38,6 @@ pool: # hacky workaround to Azure Pipelines limitations :-( steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update -qq && apt install -qy git make && make init && make ci test" +- script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' + displayName: docker build From c1fcb42ef589f0d3041ce1f4ad86590070e72e74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 0878/2295] updated bitbucket-pipelines.yml --- bitbucket-pipelines.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index f44aee45d..a7ec1b183 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -23,4 +23,7 @@ pipelines: default: - step: script: - - apt update -qq && apt install -qy git make && make init && make ci test + - setup/ci_bootstrap.sh + - make init + - make ci + - make test From fe3d147ffeae2b83816cf6f0cd16854936d3909b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 0879/2295] updated buddy.yml --- buddy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buddy.yml b/buddy.yml index 533d34910..aed9fc3bc 100644 --- a/buddy.yml +++ b/buddy.yml @@ -33,10 +33,10 @@ # - apt update # - apt install -qy git make execute_commands: - - apt update - - apt install -qy git make + - setup/ci_bootstrap.sh - make init - - make ci test + - make ci + - make test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From 32ffbefca1731eabcbf00a18a007aaa8b5a46faa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 0880/2295] updated codefresh.yml --- codefresh.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codefresh.yml b/codefresh.yml index 70990129d..542c323d8 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -33,7 +33,7 @@ steps: arguments: image: 'ubuntu:18.04' commands: - - apt update -qq && apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From f12a652de2747653a75fc4da587b3dc24eb7e538 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 0881/2295] updated shippable.yml --- shippable.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 43f564753..0c19cb053 100644 --- a/shippable.yml +++ b/shippable.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -33,6 +33,7 @@ build: - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make + - setup/ci_bootstrap.sh - make init - make ci - make test From d3f580ea68ef3a99d6798acc2d54f21703edad06 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 0882/2295] updated wercker.yml --- wercker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wercker.yml b/wercker.yml index 18b14ed8f..f57ab989f 100644 --- a/wercker.yml +++ b/wercker.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -20,8 +20,8 @@ box: debian build: steps: - script: - name: install git & make - code: apt-get update -qq && apt-get install -qy git make + name: ci bootstrap + code: setup/ci_bootstrap.sh - script: name: init code: make init From 07b68156b6cf929d9e0e265635cabe72a70bbd29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:46:10 +0100 Subject: [PATCH 0883/2295] added boot --- boot | 1 + 1 file changed, 1 insertion(+) create mode 120000 boot diff --git a/boot b/boot new file mode 120000 index 000000000..4092e5539 --- /dev/null +++ b/boot @@ -0,0 +1 @@ +setup/bootstrap.sh \ No newline at end of file From b6c79d0b34a4ac05399901fcaf37519c0f99d141 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:47:22 +0100 Subject: [PATCH 0884/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1dd707e62..bae152292 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1dd707e6228c9bda2bad2a684dfc3697237ed0b0 +Subproject commit bae152292cd75f72e92bce6131ff9a6b5ad42cc3 From 58706abe890d0eb04ebbffed97c23fa9e1756169 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:47:22 +0100 Subject: [PATCH 0885/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9d3c31caf..04ba0ca02 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9d3c31caf3d3efb90866ad225d3d469263d962a2 +Subproject commit 04ba0ca02b2196b45a0dc7f0d29e3d59642fd073 From a179e1e49acbdd238fc6b07f5988a177891b9d16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:59:05 +0100 Subject: [PATCH 0886/2295] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup/ci_bootstrap.sh diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh old mode 100644 new mode 100755 From 1dd1ce2ee77f0695ee914f198159f8ad32493105 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 18:00:26 +0100 Subject: [PATCH 0887/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 04ba0ca02..312908be6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 04ba0ca02b2196b45a0dc7f0d29e3d59642fd073 +Subproject commit 312908be6b409520e2ce51d05c6fc5779b0dbaf5 From a076ab35d703ef89d542487109443dabd2444509 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 Jun 2020 12:57:21 +0100 Subject: [PATCH 0888/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bae152292..f4641004f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bae152292cd75f72e92bce6131ff9a6b5ad42cc3 +Subproject commit f4641004f24d8ae99ffef0e170185350b2d391f1 From 867d9c17e5a24f7e052fc16fcb46bfc1b31d13a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 Jun 2020 12:57:21 +0100 Subject: [PATCH 0889/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 312908be6..d47683229 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 312908be6b409520e2ce51d05c6fc5779b0dbaf5 +Subproject commit d47683229b7a10ae74c4a8315befcb705c31c96a From 3aeee83c9444d5f812350fc7453d77e164144755 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 7 Jun 2020 15:17:54 +0100 Subject: [PATCH 0890/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d47683229..7050d3664 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d47683229b7a10ae74c4a8315befcb705c31c96a +Subproject commit 7050d3664e3d64dc4a6cb3fbb3a469e899a00f92 From d9f139f00678691436fdeb5a2b0b43d1b328474c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 15 Jun 2020 15:13:24 +0100 Subject: [PATCH 0891/2295] updated find_duplicate_files.py --- find_duplicate_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index b1f35742e..cbf7b082b 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -210,27 +210,27 @@ def run(self): for filepath in sorted(self.dup_filepaths): print(filepath) sys.exit(4) - print('Duplicates detected!\n') + print('Duplicates detected!') if self.dups_by_name: - print('Duplicates by name:\n') + print('\nDuplicates by name:\n') for basename in self.dups_by_name: print("--\nbasename '{0}':".format(basename)) for filepath in sorted(self.dups_by_name[basename]): print(filepath) if self.dups_by_size: - print('Duplicates by size:\n') + print('\nDuplicates by size:\n') for size in self.dups_by_size: print("--\nsize '{0}' bytes:".format(size)) for filepath in sorted(self.dups_by_size[size]): print(filepath) if self.dups_by_hash: - print('Duplicates by checksum:\n') + print('\nDuplicates by checksum:\n') for checksum in self.dups_by_hash: print("--\nchecksum '{0}':".format(checksum)) for filepath in sorted(self.dups_by_hash[checksum]): print(filepath) if self.dups_by_regex: - print('Duplicates by regex match ({0}):\n'.format(self.regex)) + print('\nDuplicates by regex match ({0}):\n'.format(self.regex)) for matching_portion in self.dups_by_regex: print("--\nregex matching portion '{0}':".format(matching_portion)) for filepath in sorted(self.dups_by_regex[matching_portion]): From 0ea610400a05c550dd2ade0bdc5c3bf14dcabeec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Jul 2020 14:00:33 +0100 Subject: [PATCH 0892/2295] updated find_duplicate_files.py --- find_duplicate_files.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index cbf7b082b..9fa274f26 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -35,6 +35,8 @@ then will treat the entire regex as the capture. Regex is case insensitive by default and applies only to the file's basename +Exits with exit code 4 if duplicates are found + Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of these options then the others will not run unless also explicitly specified. From e57b02628a4ec1fdd91bccc358190c5bc4f2c346 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 18 Jul 2020 17:38:07 +0100 Subject: [PATCH 0893/2295] updated find_duplicate_files.py --- find_duplicate_files.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 9fa274f26..4e382fa27 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -49,10 +49,12 @@ - By default this program will short-circuit to stop processing a file as soon as it is determined to be a duplicate file via one of the above methods in that order for efficiency. This means that if 2 files have duplicate names, and a third has a different name but the same checksum as the second one, the second one's size + checksum will not have -been recorded stored and so the third duplicate will not be detected. However, if you removed one duplicate the next -run of this program would find the other duplicate via the other dimension of checking. Given it's a rare condition -it's probably not worth the extra overhead in everyday use but this behaviour can be overridden by specifying the ---no-short-circuit option too run every check on every file. Be aware this will slow down the process. +been checked and so a third duplicate with a different name will not be detected by size / checksum. In most cases this +is a good thing to finish quicker and avoid unnecessary checksumming which is computationally expensive and time +consuming for large files. If you remove one duplicate then the next run of this program would find the other +duplicate via the additional checks of size and checksumming. Given it's a rare condition it's probably not worth the +extra overhead in everyday use but this behaviour can be overridden by specifying the --no-short-circuit option to run +every check on every file. Be aware this will slow down the process. To see progress of which files are matching size, backtracking to hash them for comparison etc use --verbose twice or -vv. To see which files are being checked use triple verbose mode -vvv From 748630ae42cbb9d85d6dce305ae3143d9db79450 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jul 2020 12:54:25 +0100 Subject: [PATCH 0894/2295] updated find_duplicate_files.py --- find_duplicate_files.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 4e382fa27..6f235fadb 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -41,6 +41,9 @@ an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of these options then the others will not run unless also explicitly specified. +If you want to find files that are probably the same by byte count but may not have the same checksum due to minor +corruption, such as large media files, then specify --size but do not specify --checksum which supercedes it + Caveats: - The limitation of the checksum approach is that it can't determine files as duplicates if there is any From 4302de0fe0e18d35e237eed93da382c8fc6be74e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:39:56 +0100 Subject: [PATCH 0895/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 305b8b1d6..89369eb58 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 200+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 300+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Kafka, Docker, Hadoop, SQL, Athena, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, Spotify API & MP3 tools, Git, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... From 2b776e2864bddb83a7cda345a0f1294319e60c27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:40:00 +0100 Subject: [PATCH 0896/2295] updated anonymize.py --- anonymize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/anonymize.py b/anonymize.py index bf87520a6..339376c13 100755 --- a/anonymize.py +++ b/anonymize.py @@ -83,6 +83,7 @@ user_regex # used dynamically # pylint: disable=unused-import + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend from harisekhon.utils import \ domain_regex, \ email_regex, \ From a60f16545e31f2056d696506c091e313072665ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:41:07 +0100 Subject: [PATCH 0897/2295] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 89369eb58..f146e8e6c 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,6 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### - * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 300+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Kafka, Docker, Hadoop, SQL, Athena, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, Spotify API & MP3 tools, Git, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies From e5c8797ecd284e081e51a9d26ed678b33aa7af66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:43:47 +0100 Subject: [PATCH 0898/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f4641004f..853b19466 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f4641004f24d8ae99ffef0e170185350b2d391f1 +Subproject commit 853b194664b69a12321c70a3ef3d41e190dd0ed9 From 5fdf345ef903f299f9eb69c362c997b36abb7bac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:43:47 +0100 Subject: [PATCH 0899/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7050d3664..fc127555b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7050d3664e3d64dc4a6cb3fbb3a469e899a00f92 +Subproject commit fc127555b0d8163e9949fada918fbc3f1a1f5745 From 6423b3808652936407929c1266a673b5e4d08d68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:37:45 +0100 Subject: [PATCH 0900/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f146e8e6c..208c2780f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![StarTrack](https://img.shields.io/badge/Star-Track-blue)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) From 99b24cab83c8b510e110849f78da1cbe9ab6adf5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:40:30 +0100 Subject: [PATCH 0901/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 853b19466..b9ad2ed08 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 853b194664b69a12321c70a3ef3d41e190dd0ed9 +Subproject commit b9ad2ed08f5474ca13d9cef7d60c44c34d467a46 From 8d096756919af6e0c5e4608e44352fbfb7acc2b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:40:30 +0100 Subject: [PATCH 0902/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fc127555b..55a3bc182 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fc127555b0d8163e9949fada918fbc3f1a1f5745 +Subproject commit 55a3bc1825f3433c115b759dc5ca7775933419f4 From 7a845b9395aee8083f9770e40af5234c2c94892e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 17:56:57 +0100 Subject: [PATCH 0903/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b9ad2ed08..b24bcf168 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b9ad2ed08f5474ca13d9cef7d60c44c34d467a46 +Subproject commit b24bcf168d8e95feb7a2b11a83cb6ddf85543d12 From 70182f306f969e17582b14fb9d48eb9c4a3bda75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 17:56:57 +0100 Subject: [PATCH 0904/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 55a3bc182..12a344a0b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 55a3bc1825f3433c115b759dc5ca7775933419f4 +Subproject commit 12a344a0b686dd6a2618a92f7d8d6311aed73037 From 7c12c396c193e1f0f69a7154ca96d3828d207755 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 14:32:19 +0100 Subject: [PATCH 0905/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b24bcf168..2f58c3492 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b24bcf168d8e95feb7a2b11a83cb6ddf85543d12 +Subproject commit 2f58c3492e68c7ab1f778197a451fe892827df8f From d856c134918baad561492ed9005d923522ecb686 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 14:32:20 +0100 Subject: [PATCH 0906/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 12a344a0b..8a8ed26f1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 12a344a0b686dd6a2618a92f7d8d6311aed73037 +Subproject commit 8a8ed26f1250132446a54f0db627bd14c214bfe4 From 54d86180a7fcb34c5f2656c8135a049b325444d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 15:58:30 +0100 Subject: [PATCH 0907/2295] added find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 find_missing_files_in_sequence.py diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py new file mode 100755 index 000000000..6261932c5 --- /dev/null +++ b/find_missing_files_in_sequence.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Finds missing files by numeric sequence, assuming a uniformly numbered file naming convention across files + +Files / directories are given as arguments or via standard input + +Directories are recursed + +You must only give directories that should be sharing a contiguously numbered file naming convention for each +single run of this tool + +Accounts for zero padding in numbered files + +Caveats: + +- This is more complicated than you'd first think as there are so many file naming variations so this is not the most + universally bulletproof piece of code in this repo by a long shot and may require advanced tuning --regex tuning to + match your use case + +- Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. + If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the max track position and + see if the files go that high + +- Returns globs instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough + use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print the missing + files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, such as + chapters of audiobooks eg. + + 'blah 01 - chapter 1.mp3' + 'blah 02 - chapter 2.mp3' + + so in the general case you cannot always infer suffixes, hence why it is left as globs. + Simpler single use case scripts would be better for printing explicit missing files as they can use hardcoded suffixes + +- Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases + easily by eye + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import glob +#import logging +import os +import re +import sys +import traceback +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + from harisekhon.utils import log, log_option, validate_regex, isFloat, UnknownError + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class FindMissingFiles(CLI): + + def __init__(self): + # Python 2.x + super(FindMissingFiles, self).__init__() + # Python 3.x + # super().__init__() + self.paths = [] + self.regex_default = r'(? 0: + self.paths = self.args + else: + self.paths = sys.stdin.readlines() + log_option('paths', self.paths) + + def is_included(self, path): + if not self.include: + return True + if self.include.search(path): + log.debug("including path: %s", path) + return True + return False + + def is_excluded(self, path): + if not self.exclude: + return False + if self.exclude.search(path): + log.debug("excluding path: %s", path) + return True + return False + + def run(self): + for path in self.paths: + if self.is_excluded(path): + continue + if not self.is_included(path): + continue + if not os.path.exists(path): + raise UnknownError('path not found: {}'.format(path)) + if os.path.isdir(path): + self.process_directory(directory=path) + elif os.path.isfile(path): + self.check_file(filename=path) + + def process_directory(self, directory): + for root, dirs, files in os.walk(directory, topdown=True): + for filename in files: + file_path = os.path.join(root, filename) + if not self.is_included(file_path): + continue + if self.is_excluded(file_path): + continue + self.check_file(filename=file_path) + for dirname in dirs: + dir_path = os.path.join(root, dirname) + if not self.is_included(dir_path): + continue + if self.is_excluded(dir_path): + continue + self.process_directory(directory=dir_path) + + def check_file(self, filename): + log.debug('checking file \'%s\'', filename) + match = self.regex.search(filename) + if not match: + log.debug('failed to match file \'%s\', skipping...', filename) + return + # will error out here if you've supplied your own regex without capture brackets + # or if you've got pre-captures - let this bubble to user to fix their regex + file_prefix = match.group(1) + file_number = match.group(2) + if not isFloat(file_number): + raise UnknownError('regex captured non-float for filename: {}'.format(filename)) + if file_prefix is None: + file_prefix = '' + padding = len(file_number) + file_number = int(file_number) + self.determine_missing_file_backfill(file_prefix, file_number, padding) + + def determine_missing_file_backfill(self, file_prefix, file_number, padding): + if file_number != 1: + file_number -= 1 + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding) + print(expected_last_filename_glob) + + +if __name__ == '__main__': + FindMissingFiles().main() From 251bb73fd3f739200567ccc047a52296550b033a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 16:18:12 +0100 Subject: [PATCH 0908/2295] added --fixed-suffix support --- find_missing_files_in_sequence.py | 73 +++++++++++++++++++------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 6261932c5..29bf95502 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -21,9 +21,9 @@ Files / directories are given as arguments or via standard input -Directories are recursed +Directories are recursed and their files examined for missing numbers before each one -You must only give directories that should be sharing a contiguously numbered file naming convention for each +Only supply files / directories that should be sharing a contiguously numbered file naming convention in each single run of this tool Accounts for zero padding in numbered files @@ -35,19 +35,21 @@ match your use case - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. - If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the max track position and - see if the files go that high + If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the total number of tracks + and see if the files go that high -- Returns globs instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough - use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print the missing - files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, such as - chapters of audiobooks eg. +- Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a + simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print + the missing files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, + such as chapters of audiobooks eg. - 'blah 01 - chapter 1.mp3' - 'blah 02 - chapter 2.mp3' + 'blah 01 - chapter about X.mp3' + 'blah 02 - chapter about Y.mp3' - so in the general case you cannot always infer suffixes, hence why it is left as globs. - Simpler single use case scripts would be better for printing explicit missing files as they can use hardcoded suffixes + so in the general case you cannot always infer suffixes, hence why it is left as globs. If you are sure that the + suffixes don't change then you can specify --fixed-suffix and it will infer each file's suffix as the basis for any + numerically missing files in the sequence, but if used where this is not the case, it'll generate a lot of false + positives that the default globbing mode would have handled - Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases easily by eye @@ -76,7 +78,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class FindMissingFiles(CLI): @@ -87,10 +89,11 @@ def __init__(self): # Python 3.x # super().__init__() self.paths = [] - self.regex_default = r'(? 0: + if self.args: self.paths = self.args else: self.paths = sys.stdin.readlines() @@ -183,23 +187,34 @@ def check_file(self, filename): # or if you've got pre-captures - let this bubble to user to fix their regex file_prefix = match.group(1) file_number = match.group(2) + file_suffix = match.group(3) if not isFloat(file_number): raise UnknownError('regex captured non-float for filename: {}'.format(filename)) if file_prefix is None: file_prefix = '' + if file_suffix is None: + file_suffix = '' padding = len(file_number) file_number = int(file_number) - self.determine_missing_file_backfill(file_prefix, file_number, padding) + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - def determine_missing_file_backfill(self, file_prefix, file_number, padding): + def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix): if file_number != 1: file_number -= 1 - expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() - expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) - if not glob.glob(expected_last_filename_glob): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding) - print(expected_last_filename_glob) + if self.fixed_suffix: + explicit_last_filename = '{}{:0>%(padding)s}{}' % locals() + explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) + if not os.path.isfile(explicit_last_filename): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + print(explicit_last_filename) + else: + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + print(expected_last_filename_glob) if __name__ == '__main__': From ecc55bb9eb6fcfe5e165bd52ddbd0c007f7949e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:45:16 +0100 Subject: [PATCH 0909/2295] refactored out recursion of file backtracking for larger file lists to avoid recursion limit --- find_missing_files_in_sequence.py | 79 +++++++++++++++++++------------ 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 29bf95502..e51b1ab4e 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -30,13 +30,14 @@ Caveats: -- This is more complicated than you'd first think as there are so many file naming variations so this is not the most - universally bulletproof piece of code in this repo by a long shot and may require advanced tuning --regex tuning to - match your use case +- This is more complicated than you'd first think as there are so many file naming variations that no code could ever + be universally bulletproof and will require advanced regex tuning to match your use case (this is tuned for the most + common use cases I've found but yours may vary since there are huge file naming variations between people and + environments) - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. - If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the total number of tracks - and see if the files go that high + If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like + 'mediainfo' to get the total number of tracks and see if the files go that high - Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print @@ -71,16 +72,17 @@ libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) try: - from harisekhon.utils import log, log_option, validate_regex, isFloat, UnknownError + from harisekhon.utils import log, log_option, validate_regex, isInt, UnknownError from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' +# pylint: disable=too-many-instance-attributes class FindMissingFiles(CLI): def __init__(self): @@ -89,22 +91,34 @@ def __init__(self): # Python 3.x # super().__init__() self.paths = [] - self.regex_default = r'(? 1: + file_number = self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + if self.missing_files: + print('\n'.join(reversed(self.missing_files))) + self.missing_files = [] def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix): - if file_number != 1: - file_number -= 1 - if self.fixed_suffix: - explicit_last_filename = '{}{:0>%(padding)s}{}' % locals() - explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) - if not os.path.isfile(explicit_last_filename): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - print(explicit_last_filename) + file_number -= 1 + if self.fixed_suffix: + explicit_last_filename = '{}{:0>%(padding)s}{}' % {'padding': padding} + explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) + if not os.path.isfile(explicit_last_filename): + self.missing_files.append(explicit_last_filename) else: - expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() - expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) - if not glob.glob(expected_last_filename_glob): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - print(expected_last_filename_glob) + file_number = -1 + else: + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + self.missing_files.append(expected_last_filename_glob) + else: + file_number = -1 + return file_number if __name__ == '__main__': From ce4ae46c48b297592d3980f5386505c1af89f12a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:50:29 +0100 Subject: [PATCH 0910/2295] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index e51b1ab4e..fb3998b5e 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -31,9 +31,7 @@ Caveats: - This is more complicated than you'd first think as there are so many file naming variations that no code could ever - be universally bulletproof and will require advanced regex tuning to match your use case (this is tuned for the most - common use cases I've found but yours may vary since there are huge file naming variations between people and - environments) + be universally bulletproof and will likely require advanced regex tuning to match your use case and naming convention - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like @@ -79,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' +__version__ = '0.3.1' # pylint: disable=too-many-instance-attributes @@ -195,14 +193,14 @@ def process_directory(self, directory): def check_file(self, filename): log.debug('checking file \'%s\'', filename) - match = self.regex.search(filename) + match = self.regex.search(os.path.basename(filename)) if not match: log.debug('failed to find numeric regex match for file, probably not a sequential file' + \ ', skipping \'%s\'', filename) return # will error out here if you've supplied your own regex without capture brackets # or if you've got pre-captures - let this bubble to user to fix their regex - file_prefix = match.group(1) + file_prefix = os.path.join(os.path.dirname(filename), match.group(1)) file_number = match.group(2) file_suffix = match.group(3) if not isInt(file_number): From 5baaece8aaefa3ed798b190f09e5af3ddab601b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:52:56 +0100 Subject: [PATCH 0911/2295] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index fb3998b5e..4f4382062 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -113,9 +113,9 @@ def add_options(self): '(default: "{}" )'\ .format(self.regex_default)) self.add_opt('-i', '--include', metavar='REGEX', - help='Include only files that match the given case-insensitive regex (eg. ".mp3$")') + help='Include only paths that match the given case-insensitive regex (eg. ".mp3$")') self.add_opt('-e', '--exclude', metavar='REGEX', default=self.exclude_default, - help='Exclude files that match the given case-insensitive regex (default: "{}" )'\ + help='Exclude paths that match the given case-insensitive regex (default: "{}" )'\ .format(self.exclude_default)) self.add_opt('-s', '--fixed-suffix', action='store_true', help='Assume fixed suffixes and infer explicit filenames rather than globs. The reason this ' + \ From 705ebbce495cbeed613b0efc0bd53499c6edfd8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:54:12 +0100 Subject: [PATCH 0912/2295] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 4f4382062..86f3f125b 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -113,7 +113,7 @@ def add_options(self): '(default: "{}" )'\ .format(self.regex_default)) self.add_opt('-i', '--include', metavar='REGEX', - help='Include only paths that match the given case-insensitive regex (eg. ".mp3$")') + help=r"Include only paths that match the given case-insensitive regex (eg. '\.mp3$')") self.add_opt('-e', '--exclude', metavar='REGEX', default=self.exclude_default, help='Exclude paths that match the given case-insensitive regex (default: "{}" )'\ .format(self.exclude_default)) From 00c7ed506752cbff892d33310688e02d8c516c0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:54:58 +0100 Subject: [PATCH 0913/2295] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 86f3f125b..17571602c 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -195,7 +195,7 @@ def check_file(self, filename): log.debug('checking file \'%s\'', filename) match = self.regex.search(os.path.basename(filename)) if not match: - log.debug('failed to find numeric regex match for file, probably not a sequential file' + \ + log.debug('no numeric regex match for file, probably not a sequential file' + \ ', skipping \'%s\'', filename) return # will error out here if you've supplied your own regex without capture brackets From 659dee26ecb4c7ad06c6408a86884744a3a51e4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 11:00:24 +0100 Subject: [PATCH 0914/2295] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 17571602c..6d2d8b0db 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.1' +__version__ = '0.3.2' # pylint: disable=too-many-instance-attributes @@ -92,6 +92,8 @@ def __init__(self): self.regex_default = r'(? Date: Wed, 5 Aug 2020 23:17:01 +0100 Subject: [PATCH 0915/2295] add SQL submodule --- .gitmodules | 3 +++ sql | 1 + 2 files changed, 4 insertions(+) create mode 160000 sql diff --git a/.gitmodules b/.gitmodules index 37537e726..c1f646f43 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = bash-tools url = https://github.com/harisekhon/bash-tools branch = master +[submodule "sql"] + path = sql + url = https://github.com/HariSekhon/SQL-scripts diff --git a/sql b/sql new file mode 160000 index 000000000..4ffa3e74f --- /dev/null +++ b/sql @@ -0,0 +1 @@ +Subproject commit 4ffa3e74fbc8693b60dd4137d2ee83acaebf7b9a From e6ee82a47ed63e6e33f5ece667130ef0d2f72372 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Aug 2020 23:33:56 +0100 Subject: [PATCH 0916/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 208c2780f..f36c94518 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Hari Sekhon - DevOps Python Tools A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. -For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins. +See also the [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools), [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. Hari Sekhon From a8d1b18ad5c9056da74a30cde5432aa94cc4d18d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Aug 2020 23:37:05 +0100 Subject: [PATCH 0917/2295] updated bootstrap.sh --- setup/bootstrap.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index a77c27e19..6fb41f25c 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -31,7 +31,9 @@ directory="pytools" if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby + if ! type brew >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby + fi elif [ "$(uname -s)" = Linux ]; then echo "Bootstrapping Linux" if type apk >/dev/null 2>&1; then From 22898c8118bc9e56d70fcdb664212815d4903e49 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:10 +0100 Subject: [PATCH 0918/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2f58c3492..13e94aa3f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2f58c3492e68c7ab1f778197a451fe892827df8f +Subproject commit 13e94aa3fe9b2bf1f3073c031e487e447d76c0ca From f4716e3ba4ef3107f0775d5d493ea4167be51f62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:11 +0100 Subject: [PATCH 0919/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8a8ed26f1..b37934205 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8a8ed26f1250132446a54f0db627bd14c214bfe4 +Subproject commit b37934205f84f82152144f1c7f294f3223817796 From fb02b24208bf0b848e1b99266b4a78df9d9c66d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:11 +0100 Subject: [PATCH 0920/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 4ffa3e74f..f84bb7199 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 4ffa3e74fbc8693b60dd4137d2ee83acaebf7b9a +Subproject commit f84bb7199396b15c5f0cb6055217b030bf338e7c From 9d7dbe21454915080a8ad71b2d3f711a978ec48e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 0921/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 13e94aa3f..e658ea4c7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 13e94aa3fe9b2bf1f3073c031e487e447d76c0ca +Subproject commit e658ea4c750414175c4ac0d7985b50ef20011164 From 208549da3825d8193586088692c94ac21df4b131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 0922/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b37934205..7b5e797d2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b37934205f84f82152144f1c7f294f3223817796 +Subproject commit 7b5e797d2232d6c5cbfc809470dd3c3cde2533ad From be6a07ed054af2cdda4ac075430527efab46f0ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 0923/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index f84bb7199..26914f010 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit f84bb7199396b15c5f0cb6055217b030bf338e7c +Subproject commit 26914f0108bb24dec27b08516f8537b73a2aae5e From 0ed46f57b4ca4ceb70b249e6df7556f3f55562b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 0924/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e658ea4c7..04685f188 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e658ea4c750414175c4ac0d7985b50ef20011164 +Subproject commit 04685f188aa01e969dd7abbb6ba9e603cf0bae21 From 4add976e16483fb64722e611095d7d942a10cbce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 0925/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7b5e797d2..78bd9caa4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7b5e797d2232d6c5cbfc809470dd3c3cde2533ad +Subproject commit 78bd9caa4475e9979a28bcf85d290e9993bc918e From f714f071887bac415d97661ab42e71d2aa40538c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 0926/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 26914f010..9736c03eb 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 26914f0108bb24dec27b08516f8537b73a2aae5e +Subproject commit 9736c03eb8b1d8e45a171267065d888b71c8281f From 0014e40f22499e4c623bdaf4bc079427e380563f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 17:19:01 +0100 Subject: [PATCH 0927/2295] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f36c94518..24c737745 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[https://www.linkedin.com/in/harisekhon](https://www.linkedin.com/in/harisekhon) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 83f9f1c6a7d3d7bad3bdb6c8aaf8a359da534a92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Aug 2020 18:18:01 +0100 Subject: [PATCH 0928/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24c737745..5ded52f24 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Hari Sekhon - DevOps Python Tools -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) From 9038af25597b9275951669bada0f449a31029c1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:43:04 +0100 Subject: [PATCH 0951/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b7a46d5a2..7bf60c2f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b7a46d5a21cd5150885c232d878dfeb147fe5719 +Subproject commit 7bf60c2f254f327252caeb060521826c60a0c376 From eb3ddb39e198efbe83c30c7525e6a2fdf7ef17a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:43:05 +0100 Subject: [PATCH 0952/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 264e67054..8992cfebe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 264e67054c081ce2c09e0abd7c1f00f7a2155de2 +Subproject commit 8992cfebee776189cd3d8c16edc53a2953846fcf From d57ba74495b6c49012d49a6263679ed687ad1630 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:49:29 +0100 Subject: [PATCH 0953/2295] updated README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 957224c6b..e8130e3b7 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,16 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) - -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) + + +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 6cd9b28757c28d4f567e4296f278b020c19ea32f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:54:11 +0100 Subject: [PATCH 0954/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7bf60c2f2..7d19d8c87 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7bf60c2f254f327252caeb060521826c60a0c376 +Subproject commit 7d19d8c876a87f7e87ba852f231abd7304eb89ba From 19268827dcf9ae7465ee41579b8350f2c824c44b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:54:11 +0100 Subject: [PATCH 0955/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8992cfebe..f40cc4ae0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8992cfebee776189cd3d8c16edc53a2953846fcf +Subproject commit f40cc4ae05a34bb35197a22f0f32d8e11ad87607 From 18e9d7f0015a7f282f145cd5a5c17982b7968296 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:05:33 +0100 Subject: [PATCH 0956/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8130e3b7..16e5b8849 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ Hari Sekhon - DevOps Python Tools [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) @@ -50,6 +49,7 @@ Hari Sekhon - DevOps Python Tools [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 44b1291e95d17624146cab6c8fce47d41fc4a841 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:08:25 +0100 Subject: [PATCH 0957/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7d19d8c87..0931d0e8a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7d19d8c876a87f7e87ba852f231abd7304eb89ba +Subproject commit 0931d0e8aa23f507b46c8f89aa6c7006473fda60 From b4642d1ddbf4a860580d46f54291f32e29875d56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:08:25 +0100 Subject: [PATCH 0958/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f40cc4ae0..5ccf87c55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f40cc4ae05a34bb35197a22f0f32d8e11ad87607 +Subproject commit 5ccf87c552d5ff08bfd6a8b20b2a2dfc990206e6 From 2a361ece594cca76f690891179532ea18bb66f42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:15:57 +0100 Subject: [PATCH 0959/2295] updated README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 16e5b8849..440c2dca3 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) + [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) @@ -46,11 +51,6 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From dfd2e71edc162b239a036949ee3e0b17eb198384 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 0960/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0931d0e8a..d0d7ac2e5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0931d0e8aa23f507b46c8f89aa6c7006473fda60 +Subproject commit d0d7ac2e54f0bc15988666511a5cde3e6f2c6f19 From 3b0fbf9e9d6ff9a77d1cd90bc34187e4bfc9c0f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 0961/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5ccf87c55..1914a9a16 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5ccf87c552d5ff08bfd6a8b20b2a2dfc990206e6 +Subproject commit 1914a9a16d6958c7d1d27b30c9b3d1170c1879d5 From 3fe7530ad6d9cbd8abd3f6a5f13dd44d4ea0485e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 0962/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8fc45acbf..ac4bd345b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8fc45acbf1346efed706804a042043f98deba060 +Subproject commit ac4bd345bf12263b87a23c9caeaca99ec849bc0d From c1607b90a8a8a1a2dab399af1c1defefe598ef51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:44:27 +0100 Subject: [PATCH 0963/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 440c2dca3..05d005769 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis%20CI)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) From 5913af68ea7a5ad939c21ef1f38c51d25e248808 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:52:08 +0100 Subject: [PATCH 0964/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05d005769..8d9e9163b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Hari Sekhon - DevOps Python Tools [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) From 62f2e6b9409c6c76f9b62a12198651ace2adcdc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:53:45 +0100 Subject: [PATCH 0965/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d9e9163b..97bd3dfb3 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Hari Sekhon - DevOps Python Tools [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) From 0072e2befa72b9e0bdfc2b51be5905ad5f935bcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:55:04 +0100 Subject: [PATCH 0966/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d0d7ac2e5..85c2eb9fe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d0d7ac2e54f0bc15988666511a5cde3e6f2c6f19 +Subproject commit 85c2eb9fe9bc7c5bc671c088ba27307f84bfc493 From d173e203ef9f56e497b8f043aa11a85aea33e9f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:55:04 +0100 Subject: [PATCH 0967/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1914a9a16..d0f5ba5a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1914a9a16d6958c7d1d27b30c9b3d1170c1879d5 +Subproject commit d0f5ba5a5e75465c86b61e2e02a0a69157b89be4 From f3243cd8c24799d504e0723b8e0cfa3422013be4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 19:15:58 +0100 Subject: [PATCH 0968/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97bd3dfb3..2c6fe394b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) From 60ed780d90938032be52ed06bc0564760d6ebe71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:34:58 +0100 Subject: [PATCH 0969/2295] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c6fe394b..1420b8898 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,11 @@ Hari Sekhon - DevOps Python Tools [![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker)](https://hub.docker.com/r/harisekhon/github/) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/) +[![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) + [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) From 1cdeb0bff9a351d7055f88baa9b194b6803eccf3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 0970/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 85c2eb9fe..6f2d0b5f7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 85c2eb9fe9bc7c5bc671c088ba27307f84bfc493 +Subproject commit 6f2d0b5f785fcdcaff8dfa110fc1017e131971cf From 56be603c68659d0dd249846754c88b4752f5f460 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 0971/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d0f5ba5a5..61ba9c321 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d0f5ba5a5e75465c86b61e2e02a0a69157b89be4 +Subproject commit 61ba9c3219d5392a46bd2ee44d485205da507ca4 From 4cd1521d80b8a41aba51e786def8fd7ea890845b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 0972/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ac4bd345b..5ef8c119f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ac4bd345bf12263b87a23c9caeaca99ec849bc0d +Subproject commit 5ef8c119f9c2a8158e034c9b84a1489e82989db0 From 52da799e1fbd6c3a58130008a3fd1203b02fe936 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:54:48 +0100 Subject: [PATCH 0973/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1420b8898..4c5047983 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) From 338e0a29da8cc30e9b5db38f4aca5736a3924b97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 13:00:05 +0100 Subject: [PATCH 0974/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c5047983..29f51beb4 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) From 4bd76109b4770c8de269ce02ae79ca5cbd3d26ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Aug 2020 19:35:31 +0100 Subject: [PATCH 0975/2295] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 29f51beb4..7410ac48f 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) - +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) -[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) + +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) From 4368fcd719288001094f85cd1dc65f892049b560 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1110/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1c4da27cc..d95f2fc67 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1c4da27ccd02c6948d669afbc8dc1e11c261d488 +Subproject commit d95f2fc67f1b0c45a752a0deeb44a257f80a8e26 From ff9de7269f6f48dbdb46f3adff3426c35ac00e57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1111/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d3cde9c87..b47a118cd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d3cde9c87741dd472ff99d72e8bcfc0e93dc504d +Subproject commit b47a118cdfa3598880d2f8a5df93e4afa9fc21c4 From e450003b73b932fc1dc717c712fe04690b038b97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1112/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 90847ef41..c83e4a492 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 90847ef411d03bcb3e7f13070d5bedf556148d6b +Subproject commit c83e4a492fbd977d9f6db3841221689d2c49306a From 8c782ad356676c75d241964f46dbd77335a13666 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1113/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7d2913a4b..6f58cf3a1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7d2913a4bf2e65ce6370f754e22638e0b4f9a06d +Subproject commit 6f58cf3a1c329b64419002ecb29bcda768a3e80b From abb0a21a3588e2846e0c07ed91d7efd0cd981eda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 26 Oct 2020 19:54:44 +0000 Subject: [PATCH 1114/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d95f2fc67..a6913bda8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d95f2fc67f1b0c45a752a0deeb44a257f80a8e26 +Subproject commit a6913bda884714f5a489337e32128d3701327b38 From 69cee93beafffe84196b964b6e0917b14d333224 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 26 Oct 2020 19:54:44 +0000 Subject: [PATCH 1115/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f58cf3a1..03efa398c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f58cf3a1c329b64419002ecb29bcda768a3e80b +Subproject commit 03efa398c177cf8d6f2fde6fdace6c686f2f95a9 From 7a340d99e893a44f1e2c6df5cc03a66422775af3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 18:15:10 +0000 Subject: [PATCH 1116/2295] updated requirements.txt --- gcp_cloud_function_sql_export/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_sql_export/requirements.txt b/gcp_cloud_function_sql_export/requirements.txt index 47e401b6d..2f7349f4d 100644 --- a/gcp_cloud_function_sql_export/requirements.txt +++ b/gcp_cloud_function_sql_export/requirements.txt @@ -1,3 +1,3 @@ # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python -google-api-python-client -Oauth2client +google-api-python-client==1.12.5 +oauth2client==4.1.3 From 967d20b36bd5c425617b3a3d4a41b55ec959a48b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:02:42 +0000 Subject: [PATCH 1117/2295] added gcp_service_account_credential_keys.py --- gcp_service_account_credential_keys.py | 197 +++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100755 gcp_service_account_credential_keys.py diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py new file mode 100755 index 000000000..6435fb3ef --- /dev/null +++ b/gcp_service_account_credential_keys.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Lists all service account credential keys in a given GCP project + +Excludes built-in system managed keys which are hidden in the Console UI anyway and are not actionable +or in scope for a key policy audit. + + +Output Format: + + + + +You can supply a service account credentials file to authenticate with or just use ADC via: + + gcloud auth application-default login + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from datetime import datetime +import json +import os +import sys +import traceback +from google.oauth2 import service_account +import googleapiclient.discovery +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, log_option, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class GcpServiceAccountCredentialKeys(CLI): + + def __init__(self): + # Python 2.x + super(GcpServiceAccountCredentialKeys, self).__init__() + # Python 3.x + # super().__init__() + self.credentials_file = None + self.service = None + self.project = None + self.no_expiry = None + self.expired = None + self.expires_within_days = None + + def add_options(self): + super(GcpServiceAccountCredentialKeys, self).add_options() + self.add_opt('-f', '--credentials-file', metavar='', + default=os.getenv('GOOGLE_CREDENTIALS', \ + os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + help='Credentials file ($GOOGLE_CREDENTIALS, ' + \ + '$GOOGLE_APPLICATION_CREDENTIALS)') + self.add_opt('-p', '--project-id', metavar='', + help='Google Cloud Project ID ($GOOGLE_PROJECT_ID, or inferred from credentials file') + self.add_opt('-n', '--no-expiry', action='store_true', help='List only non-expiring keys') + self.add_opt('-e', '--expired', action='store_true', help='List only expired keys') + self.add_opt('-d', '--expires-within-days', type=int, help='List only keys that will expire within N days') + + def process_options(self): + super(GcpServiceAccountCredentialKeys, self).process_options() + self.no_args() + project_id = self.get_opt('project_id') + credsfile = self.get_opt('credentials_file') + self.no_expiry = self.get_opt('no_expiry') + self.expired = self.get_opt('expired') + self.expires_within_days = self.get_opt('expires_within_days') + #if not credsfile: + # self.usage('no --credentials-file given and ' + \ + # 'GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS environment variables not populated') + if credsfile: + if not os.path.exists(credsfile): + self.usage('credentials file not found: {}'.format(credsfile)) + self.credentials_file = credsfile + log_option('credentials file', self.credentials_file) + if not project_id: + if credsfile: + json_data = json.loads(open(credsfile).read()) + project_id = json_data['project_id'] + else: + self.usage('--project-id not specified and no credentials file given from which to infer') + self.project = project_id + log_option('project', self.project) + if self.expires_within_days is not None: + validate_int(self.expires_within_days, 'expires within days', 0) + if self.no_expiry: + self.usage('--expires-within-days and --no-expiry are mutually exclusive') + if self.expired: + self.usage('--expires-within-days and --expired are mutually exclusive') + if self.no_expiry and self.expired: + self.usage('--expired and --no-expiry are mutually exclusive') + + def run(self): + # defaults to looking for $GOOGLE_APPLICATION_CREDENTIALS or using Application Default Credentials + # from 'gcloud auth application-default login' => ~/.config/gcloud/application_default_credentials.json + credentials = None + if self.credentials_file: + log.debug('loading credentials') + credentials = service_account.Credentials.from_service_account_file( + filename=self.credentials_file, + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + + # cache_discovery=False avoids: + # ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth + self.service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False) + + for service_account_email in self.get_service_accounts(): + self.list_keys(service_account_email) + + def get_service_accounts(self): + """ Returns a list of service account email addresses """ + + log.debug('getting service accounts') + service_accounts = self.service.projects()\ + .serviceAccounts()\ + .list(name='projects/{}'.format(self.project))\ + .execute() + for account in service_accounts['accounts']: + yield account['email'] + + def list_keys(self, service_account_email): + log.debug("getting keys for service account '%s'", service_account_email) + keys = self.service.projects()\ + .serviceAccounts()\ + .keys()\ + .list(name='projects/-/serviceAccounts/' + service_account_email).execute() + + for key in keys['keys']: + if key['keyType'] == 'SYSTEM_MANAGED': + continue + _id = key['name'].split('/')[-1] + created_date = key['validAfterTime'] + expiry_date = key['validBeforeTime'] + created_datetime = datetime.strptime(created_date, "%Y-%m-%dT%H:%M:%SZ") + expiry_datetime = datetime.strptime(expiry_date, "%Y-%m-%dT%H:%M:%SZ") + age_timedelta = datetime.utcnow() - created_datetime + age_days = int(age_timedelta.total_seconds() / 86400) + expired = False + if expiry_date == '9999-12-31T23:59:59Z': + expires_in_days = 'NEVER' + else: + expires_in_timedelta = expiry_datetime - datetime.utcnow() + expires_in_days = int(expires_in_timedelta.total_seconds() / 86400) + if expires_in_days < 1: + expired = True + if self.no_expiry and (expires_in_days != 'NEVER'): + continue + if self.expired and expired: + continue + if self.expires_within_days is not None and \ + (expires_in_days == 'NEVER' or expires_in_days > self.expires_within_days): + continue + print('{id} {created} {expires} {age:4d} {expires_in:5s} {expired} {service_account}'.format( + id=_id, + created=created_date, + expires=expiry_date, + age=age_days, + expires_in=expires_in_days, + expired=expired, + service_account=service_account_email + )) + + +if __name__ == '__main__': + GcpServiceAccountCredentialKeys().main() From ff83cdd85079221f453535ebc686471be7d3c078 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:04:25 +0000 Subject: [PATCH 1118/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 38944cc4d..10ea2c017 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 1b62ae4956490739af7d35089b19ee7ea8e52341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:05:03 +0000 Subject: [PATCH 1119/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 10ea2c017..6f339e6dd 100644 --- a/README.md +++ b/README.md @@ -170,8 +170,8 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` From a4492d393420c6b7d0e8c7dd4d2ffc438a79cbf9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:06:18 +0000 Subject: [PATCH 1120/2295] updated gcp_service_account_credential_keys.py --- gcp_service_account_credential_keys.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py index 6435fb3ef..bd4460f35 100755 --- a/gcp_service_account_credential_keys.py +++ b/gcp_service_account_credential_keys.py @@ -32,6 +32,11 @@ gcloud auth application-default login + +See Also - similar scripts in the DevOps Bash tools repo: + + https://github.com/HariSekhon/DevOps-Bash-tools/ + """ from __future__ import absolute_import From 39d1ba5b1a6f9a2b395fe3e941f988a49022e994 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 2 Nov 2020 10:29:14 +0000 Subject: [PATCH 1121/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f339e6dd..45c41ab21 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From cf441dcac8f0e24964b35ae81bfe906df7f8217d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Nov 2020 18:53:08 +0000 Subject: [PATCH 1122/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45c41ab21..de5c8ebd5 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 450+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 500+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From 9c6db0c6c51a2e39ad0be12d5cb5fe6086a966ca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:55 +0000 Subject: [PATCH 1123/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a6913bda8..42b187dd8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a6913bda884714f5a489337e32128d3701327b38 +Subproject commit 42b187dd8150fcdb872d5b07191129202e6bbe2d From 0ebfc991e793d3cef90f1b3b1ee96415e0ef23a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1124/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b47a118cd..3338b29be 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b47a118cdfa3598880d2f8a5df93e4afa9fc21c4 +Subproject commit 3338b29be1de01b2f72677e868eccc19bdec0bbc From c4e8f98c64b85bc5b2734753926bc1b66ffb7cf8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1125/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index c83e4a492..8039c0aba 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit c83e4a492fbd977d9f6db3841221689d2c49306a +Subproject commit 8039c0abae11bc4595dea5af9c9a7e237015cc94 From f066926deee69c073ff3c93a011cf7b24caf7459 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1126/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 03efa398c..9a2a821bd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 03efa398c177cf8d6f2fde6fdace6c686f2f95a9 +Subproject commit 9a2a821bd85d3bae4524f16164cae03039c1a4e3 From 0754cfa6761a78b7bd56ec77c58116655ff96e83 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 14:26:32 +0000 Subject: [PATCH 1127/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 42b187dd8..a03cc5415 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 42b187dd8150fcdb872d5b07191129202e6bbe2d +Subproject commit a03cc54157363a0212f220eedbdce5714bd66f1e From c0206ba5e449c26c836ee04cdd4f0e9e7d2f557f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 14:26:33 +0000 Subject: [PATCH 1128/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3338b29be..6e250073e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3338b29be1de01b2f72677e868eccc19bdec0bbc +Subproject commit 6e250073e5efe1110cdb43630b1778e5c8db4776 From 297e6d6d6e2e3ae50fbffcfec84b74fd473b30d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Dec 2020 23:12:09 +0000 Subject: [PATCH 1129/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de5c8ebd5..f76dfe6f0 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 500+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From 963a7ed903c1c0ae7cf6eab42d00287430cc8024 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Dec 2020 23:52:19 +0000 Subject: [PATCH 1130/2295] updated shippable.yml --- shippable.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shippable.yml b/shippable.yml index 0c19cb053..71c6c3c76 100644 --- a/shippable.yml +++ b/shippable.yml @@ -32,6 +32,9 @@ build: # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* + # Basho repo is giving a '402 payment required' error + # https://github.com/Shippable/support/issues/5172 + - rm -fv /etc/apt/sources.list.d/basho_riak.list #- shippable_retry make - setup/ci_bootstrap.sh - make init From 63d073f5cb280ea07743e1a390b10c282cf3a79b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 4 Dec 2020 23:20:21 +0000 Subject: [PATCH 1131/2295] updated test_anonymize.sh --- tests/test_anonymize.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 53b219274..713b844ea 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -546,6 +546,8 @@ fi # this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle #for (( i = 0 ; i < ${#src[@]} ; i++ )); do run_tests(){ + # expands to the list of indicies in the array, starting at zero - this is easier to work with that ${#src} which is a total + # that is off by one for index usage and doesn't support sparse arrays for any missing/disabled test indicies test_numbers="${*:-${!src[*]}}" for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } From e98fef5b5819c89c203b05335bf3c01681db62b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 5 Dec 2020 21:11:30 +0000 Subject: [PATCH 1132/2295] updated .travis.yml --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf0870f49..fcdaf693a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,10 @@ python: - "pypy" # currently Python 2.7.13, PyPy 7.1.1 - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 +# https://docs.travis-ci.com/user/reference/osx/ +# macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ +osx_image: xcode12.2 + matrix: fast_finish: true include: @@ -71,7 +75,6 @@ matrix: language: python python: "pypy3" - # allow_failures: # - python: "3.4" # - python: "3.5" @@ -81,8 +84,6 @@ matrix: # - python: "pypy" # - python: "pypy3" -#dist: trusty - sudo: required env: From f82255dc507bdfca6268197c2f5f18ac6648f046 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 5 Dec 2020 22:55:32 +0000 Subject: [PATCH 1133/2295] updated .travis.yml --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcdaf693a..fcd6630b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,10 +30,6 @@ python: - "pypy" # currently Python 2.7.13, PyPy 7.1.1 - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 -# https://docs.travis-ci.com/user/reference/osx/ -# macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ -osx_image: xcode12.2 - matrix: fast_finish: true include: @@ -44,6 +40,9 @@ matrix: - os: osx language: generic # workaround since Mac doesn't have Python support yet, so install to system Python + # https://docs.travis-ci.com/user/reference/osx/ + # macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ + osx_image: xcode12.2 - os: linux language: python From 20428ff8bdc2943bca54875dcdbf89e71a0c00ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Dec 2020 23:27:27 +0000 Subject: [PATCH 1134/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f76dfe6f0..b4a8b8480 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts) +[![StarCharts](https://img.shields.io/badge/Star-Charts-blue?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md) From eb7242dbfbd8b2e78009c13148763081325c1d94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Jan 2021 18:09:46 +0000 Subject: [PATCH 1190/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9c47e9ece..46da98469 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9c47e9ece389cf07c29d414bfb652059cb93b2c6 +Subproject commit 46da98469a78ac68e835b4f72088dcb6ad8bd11c From 589cb2303b6262ee3d2f10c554b458b2bb86d49d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Jan 2021 18:09:46 +0000 Subject: [PATCH 1191/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f5ceb0e08..204fe01f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f5ceb0e0840ba35170277e3d978eae2187063be5 +Subproject commit 204fe01f4ed7c0bd5f8a83fc972ae4589b43bdd4 From afc44f30d4663f7c323aec108e1dafe64ea38203 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:02 +0000 Subject: [PATCH 1192/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46da98469..01b9badbb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46da98469a78ac68e835b4f72088dcb6ad8bd11c +Subproject commit 01b9badbb5c237f07b2dce1108f606be4a166dfa From 3de9f649db081b79955d8c79f8b85b7b8252e5f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:02 +0000 Subject: [PATCH 1193/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 204fe01f4..efb7245b3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 204fe01f4ed7c0bd5f8a83fc972ae4589b43bdd4 +Subproject commit efb7245b32876d494e9c02ac718277a0e4c0c0b5 From cc345093f15e758215b46a1f8c9a1c8e0093d130 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:03 +0000 Subject: [PATCH 1194/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c42667ad8..5ddab4747 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c42667ad8423aeb06d8222c76cd4d120ad30f4ee +Subproject commit 5ddab474791ca70ff9870373584dc9c65c017ba8 From 1c60303d95009247bb0f2024cd5e5edf080ef01a Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 21 Jan 2021 06:39:28 +0000 Subject: [PATCH 1195/2295] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-PYYAML-590151 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..d7fd0b720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,3 +56,4 @@ thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 +pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability From 62da28abc3344cac236dce723a837d9859e13b7b Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 2 Feb 2021 06:40:04 +0000 Subject: [PATCH 1196/2295] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..dadd75a0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 -Jinja2==2.10.1 +Jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 From 366bf335996364f104706729487103c977addb6b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 15:57:43 +0000 Subject: [PATCH 1197/2295] renamed .teamcity.vcs.auth.json to .teamcity.vcs.oauth.json --- .teamcity.vcs.auth.json => .teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.auth.json => .teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.auth.json b/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.auth.json rename to .teamcity.vcs.oauth.json From 32f98ca58eae04ebcbe2ecb74509059e47a901a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 15:59:50 +0000 Subject: [PATCH 1198/2295] added .teamcity.vcs.ssh.json --- .teamcity.vcs.ssh.json | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .teamcity.vcs.ssh.json diff --git a/.teamcity.vcs.ssh.json b/.teamcity.vcs.ssh.json new file mode 100644 index 000000000..1b780675a --- /dev/null +++ b/.teamcity.vcs.ssh.json @@ -0,0 +1,65 @@ +{ + "id": "TeamCity", + "name": "TeamCity", + "vcsName": "jetbrains.git", + "href": "/app/rest/vcs-roots/id:TeamCity", + "project": { + "id": "_Root", + "name": "", + "description": "Contains all other projects", + "href": "/app/rest/projects/id:_Root", + "webUrl": "http://localhost:8111/project.html?projectId=_Root" + }, + "properties": { + "count": 11, + "property": [ + { + "name": "agentCleanFilesPolicy", + "value": "ALL_UNTRACKED" + }, + { + "name": "agentCleanPolicy", + "value": "ON_BRANCH_CHANGE" + }, + { + "name": "authMethod", + "value": "TEAMCITY_SSH_KEY" + }, + { + "name": "branch", + "value": "refs/heads/master" + }, + { + "name": "ignoreKnownHosts", + "value": "true" + }, + { + "name": "submoduleCheckout", + "value": "CHECKOUT" + }, + { + "name": "teamcitySshKey", + "value": "VCS SSH Key" + }, + { + "name": "url", + "value": "github.com:HariSekhon/TeamCity-CI" + }, + { + "name": "useAlternates", + "value": "true" + }, + { + "name": "username", + "value": "git" + }, + { + "name": "usernameStyle", + "value": "USERID" + } + ] + }, + "vcsRootInstances": { + "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" + } +} From 6f0fa2482bf91791702f266b92df47732822b315 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:43 +0000 Subject: [PATCH 1199/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 01b9badbb..9410e2913 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 01b9badbb5c237f07b2dce1108f606be4a166dfa +Subproject commit 9410e29130a0567a78d0f6e24a33b13f950a32d0 From 63a6fc04cdd4233d374fbdb9ac750bd347eb48db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:43 +0000 Subject: [PATCH 1200/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index efb7245b3..cefc87fbb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit efb7245b32876d494e9c02ac718277a0e4c0c0b5 +Subproject commit cefc87fbb8278156ffaec1c2930cdba66e55c4ad From 7c98f118f9d9db843a570c9797cec3f0a7a0c52b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:44 +0000 Subject: [PATCH 1201/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ad0d35a9a..2f8bd447f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ad0d35a9a2417a8cbdef65c0f9cbbe023394d408 +Subproject commit 2f8bd447f236a4875a161f41ac30684b227acdf6 From 40fb56c4241b612854f9ecd8c1bfff0f4a40b80e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:44 +0000 Subject: [PATCH 1202/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5ddab4747..f6dd9a3d0 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5ddab474791ca70ff9870373584dc9c65c017ba8 +Subproject commit f6dd9a3d0b5403949b01f0a2f13f0d3c4c894b02 From 468976a7fdd6e5f7a416ac3666831d886e540330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:37:03 +0000 Subject: [PATCH 1203/2295] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 29718e8b3..691e04a2f 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -39,7 +39,7 @@ retry(){ "$@" && break; echo - if [ $count -ge $max_tries ]; then + if [ "$count" -ge "$max_tries" ]; then echo "$count tries failed, aborting..." exit 1 fi From f0f8a293ee18d04f218b3d3a0802ca5606a51192 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:06 +0000 Subject: [PATCH 1204/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9410e2913..3bf5112f4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9410e29130a0567a78d0f6e24a33b13f950a32d0 +Subproject commit 3bf5112f4efa8664f64dc37cedb0c3c130d90356 From 9edc5afa50931e8e9d1ed5550ac327361ba4c256 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:06 +0000 Subject: [PATCH 1205/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cefc87fbb..3d2bd6438 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cefc87fbb8278156ffaec1c2930cdba66e55c4ad +Subproject commit 3d2bd6438b0cf6fcc1ea5c47ff0b7140edd2fef1 From 02bbb77a5b9bf91baa742a6881ff6bf2f42b25f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:07 +0000 Subject: [PATCH 1206/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index f6dd9a3d0..58dc8e0c4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit f6dd9a3d0b5403949b01f0a2f13f0d3c4c894b02 +Subproject commit 58dc8e0c4ac85128c8f0059b6c762d4e8c163de4 From ec394d767c75f02c1bf440ece406f285a93c0ec1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:51:35 +0000 Subject: [PATCH 1207/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b22c5486b..69316433b 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Hari Sekhon - DevOps Python Tools [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 04c9a928c25af511e280ea0b778480d65057066f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1208/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3bf5112f4..0298b6224 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3bf5112f4efa8664f64dc37cedb0c3c130d90356 +Subproject commit 0298b6224d23a0648df7001d945960c9462bc4c9 From e1c534ccc8992b87a056a46e93e50e38dc9a5b57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1209/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3d2bd6438..539fd5b4d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3d2bd6438b0cf6fcc1ea5c47ff0b7140edd2fef1 +Subproject commit 539fd5b4de2a77f8f36bb19e9ee4b969dbd8d384 From 83b090d45aeb699ea8a0727c75fd3353df5ae127 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1210/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 58dc8e0c4..d57e9cac4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 58dc8e0c4ac85128c8f0059b6c762d4e8c163de4 +Subproject commit d57e9cac4960159be9887113a4c2f30b2c9bade4 From fe9ca366b127af6b42cd85b279fd18668e5d457b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 23:03:49 +0000 Subject: [PATCH 1211/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0298b6224..23cecb014 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0298b6224d23a0648df7001d945960c9462bc4c9 +Subproject commit 23cecb014fbe8d5b103d01e9fa487687c95d7859 From 8b78c3299d3f2513cc43599d15ba1b91e79e42b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 23:04:38 +0000 Subject: [PATCH 1212/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 539fd5b4d..f6eb8fa48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 539fd5b4de2a77f8f36bb19e9ee4b969dbd8d384 +Subproject commit f6eb8fa48f06cd42b692bf1a61c5b84915bdd015 From 2e460237e87eca44649762355c8a3ee5c0fea8e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:16 +0000 Subject: [PATCH 1213/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 23cecb014..c28241777 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 23cecb014fbe8d5b103d01e9fa487687c95d7859 +Subproject commit c28241777474d0291f8d5422a55a23b3a92e2b49 From ebdddf0cd33fde7cec8fb8d95b660350d19417b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1214/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f6eb8fa48..a20b1b987 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f6eb8fa48f06cd42b692bf1a61c5b84915bdd015 +Subproject commit a20b1b987cae04ec098b6b27969f185f90307430 From da16282b46f208bfbde5ad9a3f90eb9eb94a1da4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1215/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 2f8bd447f..9cefa28a2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 2f8bd447f236a4875a161f41ac30684b227acdf6 +Subproject commit 9cefa28a28962e1b6f1bb935c2a405b62bcba5d1 From 17e4a0709e2a057baa16ea782bb585a1285e3aea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1216/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d57e9cac4..dc906b935 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d57e9cac4960159be9887113a4c2f30b2c9bade4 +Subproject commit dc906b9356ed7bd8d8c8c6c3079cd7b8acf7ece7 From c18f5def8b103dba54629372989bace7668427ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 19 Mar 2021 18:44:24 +0000 Subject: [PATCH 1217/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69316433b..b8570d3ab 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) -[![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) +[![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From 592385a88f2d17e01e1a2505a39a53f2c045e3fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 12:51:50 +0100 Subject: [PATCH 1218/2295] updated Jenkinsfile --- Jenkinsfile | 150 +++++++++++++++++++--------------------------------- 1 file changed, 53 insertions(+), 97 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c9fbad7d3..c95f1d1e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,120 +14,76 @@ // // ========================================================================== // -// J e n k i n s P i p e l i n e +// J e n k i n s P i p e l i n e // ========================================================================== // +// Epic Jenkinsfile template: +// +// https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile + +// Official Documentation: +// // https://jenkins.io/doc/book/pipeline/syntax/ +// +// https://www.jenkins.io/doc/pipeline/steps/ +// +// https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/ pipeline { - // run pipeline any agent - agent any - // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' -// agent { -// docker { -// image 'ubuntu:18.04' -// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' -// } -// } + // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top + agent any - // need to specify at least one env var if enabling - //environment { - // DEBUG = '1' - //} + options { + timestamps() - options { - // put timestamps in console logs - timestamps() + timeout(time: 2, unit: 'HOURS') + } - // timeout entire pipeline after 4 hours - timeout(time: 4, unit: 'HOURS') + triggers { + cron('H 10 * * 1-5') + pollSCM('H/2 * * * *') + } - //retry entire pipeline 3 times - //retry(3) + stages { + stage ('Checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + } } - triggers { - cron('H 10 * * 1-5') - pollSCM('H/2 * * * *') - } - - stages { - stage ('Checkout') { - steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) - } + stage('Build') { + steps { + echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}" + echo 'Building...' + timeout(time: 10, unit: 'MINUTES') { + retry(3) { +// sh 'apt update -q' +// sh 'apt install -qy make' +// sh 'make init' + sh """ + setup/ci_bootstrap.sh && + make init + """ + } } - - stage('Build') { - steps { - echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" - echo 'Building...' - timeout(time: 10, unit: 'MINUTES') { - retry(3) { -// sh 'apt update -q' -// sh 'apt install -qy make' -// sh 'make init' - sh """ - setup/ci_bootstrap.sh && - make init - """ - } - } - timeout(time: 180, unit: 'MINUTES') { - sh 'make ci' - } - } + timeout(time: 180, unit: 'MINUTES') { + sh 'make ci' } - - stage('Test') { - options { - retry(2) - } - steps { - echo 'Testing...' - timeout(time: 120, unit: 'MINUTES') { - sh 'make test' - } - } - } - -// stage('Human gate') { -// steps { -// input "Proceed to deployment?" -// } -// } -// -// stage('Deployment') { -// steps { -// echo 'Deploying...' -// echo 'Nothing to deploy' -// } -// } - + } } - post { - always { - echo 'Always' - //deleteDir() // clean up workspace - // collect JUnit reports for Jenkins UI - //junit 'build/reports/**/*.xml' - // collect artifacts to Jenkins for analysis - //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true - } - success { - echo 'SUCCESS!' - } - failure { - echo 'FAILURE!' - } - unstable { - echo 'UNSTABLE!' - } - changed { - echo 'Pipeline state change! (success vs failure)' + stage('Test') { + options { + retry(2) + } + steps { + echo 'Testing...' + timeout(time: 120, unit: 'MINUTES') { + sh 'make test' } + } } + } } From 261f5e99c6a9d71a9b548534412fad3de6a9bcf0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 12:53:49 +0100 Subject: [PATCH 1219/2295] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c95f1d1e7..7739f92e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ // // ========================================================================== // -// J e n k i n s P i p e l i n e +// J e n k i n s P i p e l i n e // ========================================================================== // // Epic Jenkinsfile template: From b361f83215c5128d16283a412f9cd610038b4a21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1220/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c28241777..c13327c6e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c28241777474d0291f8d5422a55a23b3a92e2b49 +Subproject commit c13327c6ef2c1b4354fd49f559b42b7094d0a80b From 51e6794f10a7b739c0261373231a140cb914d979 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1221/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a20b1b987..41beb39b8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a20b1b987cae04ec098b6b27969f185f90307430 +Subproject commit 41beb39b8e17734cfc4598e11f5da3043a15f829 From adbb8dc480f3531d351882f89c89405c12422992 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1222/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 9cefa28a2..80e2004a1 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 9cefa28a28962e1b6f1bb935c2a405b62bcba5d1 +Subproject commit 80e2004a17865c23992f3ad8d5bae3d55e60dc46 From 32d27d5f4014f07c632c52842eff1a9129a97deb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1223/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index dc906b935..25324f745 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit dc906b9356ed7bd8d8c8c6c3079cd7b8acf7ece7 +Subproject commit 25324f745c7866a0d2990e60bec24342655e1185 From 5230833e5b877aebb3b49214f166b5fc7848233f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:18:09 +0100 Subject: [PATCH 1224/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c13327c6e..89e5adffb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c13327c6ef2c1b4354fd49f559b42b7094d0a80b +Subproject commit 89e5adffb393097a8d32ffece93bb299ac56e46b From 088780143cc0bbd61d284448bfa738af2c899307 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:18:09 +0100 Subject: [PATCH 1225/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 41beb39b8..28f571cce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 41beb39b8e17734cfc4598e11f5da3043a15f829 +Subproject commit 28f571cceaabc4e0025cd2513b70d8f5db52e1a6 From df07261681abdf0c112b20394c985bf0abf9281a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:44:40 +0100 Subject: [PATCH 1226/2295] added codeship.yml --- codeship.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml new file mode 100644 index 000000000..8b119709c --- /dev/null +++ b/codeship.yml @@ -0,0 +1,30 @@ +# +# Author: Hari Sekhon +# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this + +# https://github.com/painless-software/codeship-yaml + +--- +install: + - sudo rm -fv /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list + - sudo rm -fv /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list + - make +#before_script: +# - somecommand +script: + - make test +#after_success: +# - echo "Now we can deploy" From f105b83090ef89baa4b9680caf6bc91493bd7e42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:54:31 +0100 Subject: [PATCH 1227/2295] updated codeship.yml --- codeship.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/codeship.yml b/codeship.yml index 8b119709c..53efd9424 100644 --- a/codeship.yml +++ b/codeship.yml @@ -13,14 +13,40 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e S h i p +# ============================================================================ # + # 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this # https://github.com/painless-software/codeship-yaml +# Requires setting up the CodeShip commands like so: +# +# pip install codeship-yaml +# codeship-yaml +# +# or seaparately in sections: +# +# Project Settings > Test Settings > Setup Commands: +# +# pip install codeship-yaml +# codeship-yaml install +# +# Project Settings > Test Settings > Test Commands: +# +# codeship-yaml before_script script +# +# Project Settings > Deployment > (branch name) +# +# codeship-yaml after_success + --- install: + # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - sudo rm -fv /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - sudo rm -fv /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list + - sudo rm -fv /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - make #before_script: # - somecommand From c8ba8395b932e6be810f750e1509e6d77b5de0f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:57:00 +0100 Subject: [PATCH 1228/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 89e5adffb..bc2d191fa 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 89e5adffb393097a8d32ffece93bb299ac56e46b +Subproject commit bc2d191fad8b63df60944f902da10f3cb4d0eaea From 4ce4f7ad732c8a12f78a2a32bf7b11573e1e187c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:57:01 +0100 Subject: [PATCH 1229/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 28f571cce..65631842d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 28f571cceaabc4e0025cd2513b70d8f5db52e1a6 +Subproject commit 65631842daa333da9e8f5ca4ea26f30d380e53b6 From 1b373a178f0402fc6e9d9df2c44603db0f724389 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:35:29 +0100 Subject: [PATCH 1230/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bc2d191fa..b1363ed43 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bc2d191fad8b63df60944f902da10f3cb4d0eaea +Subproject commit b1363ed437868c8eed63d5aa628c1fee8cb676c0 From 222035289f999345b4f3009ee561618dcff8e894 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:35:30 +0100 Subject: [PATCH 1231/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 25324f745..a9ebed85a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 25324f745c7866a0d2990e60bec24342655e1185 +Subproject commit a9ebed85a6753ef025b8d1c49fdb6a466bb66727 From 0026bde0659f2affa328ee345844c08697e400f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:41:00 +0100 Subject: [PATCH 1232/2295] added dynamic targets for pyc and pyo creation --- Makefile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d939369eb..9960a1d0b 100755 --- a/Makefile +++ b/Makefile @@ -74,12 +74,24 @@ build: init init: git submodule update --init --recursive +# Magic to build the dependencies for only given program(s) +%.pyc: + @# this utility script supports taking .pyc or .pyo names and still does the right thing + bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` +%.pyo: + bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` + +.PHONY: pylib +pylib: + @$(MAKE) python-version + cd pylib && $(MAKE) + .PHONY: python -python: +python: pylib # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - @$(MAKE) python-version - cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to @if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \ From 8a1347736fcdad4b0add0d9f8841eddaf08723e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:49:21 +0100 Subject: [PATCH 1233/2295] updated Makefile --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9960a1d0b..b835f8699 100755 --- a/Makefile +++ b/Makefile @@ -75,11 +75,14 @@ init: git submodule update --init --recursive # Magic to build the dependencies for only given program(s) -%.pyc: +# +# dependency of same % stem prefix checks for a matching .py file to consider it a valid target +# +%.pyc: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` -%.pyo: +%.pyo: %.py bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` From 8207890a84d2f22f4881850579fc7e3a3e2ee697 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:04 +0100 Subject: [PATCH 1234/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b1363ed43..7b69c6157 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b1363ed437868c8eed63d5aa628c1fee8cb676c0 +Subproject commit 7b69c61571a3a27b786884999fd0937a8150cc78 From 2c6be8587928a68388f0e181ae26df1a2e8299ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:42 +0100 Subject: [PATCH 1235/2295] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b835f8699..e8ef4c60d 100755 --- a/Makefile +++ b/Makefile @@ -80,10 +80,10 @@ init: # %.pyc: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` %.pyo: %.py - bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` .PHONY: pylib From 393ec22b215d81f11c8a3a8f7e728b97e20fc5c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:51 +0100 Subject: [PATCH 1236/2295] updated README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b8570d3ab..18daac3e3 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,11 @@ cd pytools make ``` +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +``` +make anonymize.pyc +``` + Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. From 2d8338292a0c9a9b133bfa4a061330d0fb88ff7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:13:40 +0100 Subject: [PATCH 1237/2295] updated Makefile --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e8ef4c60d..ac40a686d 100755 --- a/Makefile +++ b/Makefile @@ -79,12 +79,16 @@ init: # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # %.pyc: %.py - @# this utility script supports taking .pyc or .pyo names and still does the right thing + @# this utility script supports taking .pyc or .pyo names and still does the right thing, @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ - python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` + python -m py_compile $< && \ + echo && \ + echo Generated $@ %.pyo: %.py @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ - python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` + python -O -m py_compile $< && \ + echo && \ + echo Generated $@ .PHONY: pylib pylib: From 53b26cae33944a01fe81c84a62d637b9d1037993 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 8 May 2021 13:11:25 +0100 Subject: [PATCH 1238/2295] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ac40a686d..abe77d452 100755 --- a/Makefile +++ b/Makefile @@ -78,13 +78,13 @@ init: # # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # -%.pyc: %.py +%.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing, @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ -%.pyo: %.py +%.pyo:: %.py @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ From 9e3b722b55ec604d249b109c980312b4fb4b49fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 10:44:36 +0100 Subject: [PATCH 1239/2295] added selenium_test.py --- selenium_test.py | 145 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100755 selenium_test.py diff --git a/selenium_test.py b/selenium_test.py new file mode 100755 index 000000000..3a4118e43 --- /dev/null +++ b/selenium_test.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME +against a given URL and content (defaults to google.com) + +Example: + + ./selenium_test.py --host [options] + + ./selenium_test.py --host selenium-hub FIREFOX CHROME + + ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --content google + ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --regex 'goog.*' + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +import time +import traceback +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class SeleniumTest(CLI): + + def __init__(self): + # Python 2.x + super(SeleniumTest, self).__init__() + # Python 3.x + # super().__init__() + self.host = None + self.port = None + self.protocol = 'http' + self.name = 'Selenium Hub' + self.default_port = 80 + self.path = 'wd/hub' + self.url_default = 'http://google.com' + self.url = self.url_default + self.expected_content = None + self.expected_regex = None + self.timeout_default = 600 + self.verbose_default = 2 + + def add_options(self): + super(SeleniumTest, self).add_options() + self.add_hostoption(name='Selenium Hub', default_port=4444) + self.add_opt('-u', '--url', default=self.url_default, + help='URL to use for the test (default: {})'.format(self.url_default)) + self.add_opt('-c', '--content', help='URL content to expect') + self.add_opt('-r', '--regex', help='URL content to expect') + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub') + + def process_options(self): + super(SeleniumTest, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + self.url = self.get_opt('url') + if ':' not in self.url: + self.url = 'http://' + self.url + self.expected_content = self.get_opt('content') + self.expected_regex = self.get_opt('regex') + if self.expected_regex: + validate_regex(self.expected_regex) + self.expected_regex = re.compile(self.expected_regex) + validate_host(self.host) + validate_port(self.port) + validate_url(self.url) + if self.get_opt('ssl'): + self.protocol = 'https' + if not self.args: + self.usage() + + def check_selenium(self, capability, url): + selenium_url = '{protocol}://{host}:{port}/{path}'\ + .format(protocol=self.protocol, \ + host=self.host, \ + port=self.port, \ + path=self.path) + log.info("Connecting to '%s' with capability '%s'", selenium_url, capability) + driver = webdriver.Remote( + command_executor=selenium_url, + desired_capabilities=getattr(DesiredCapabilities, capability) + ) + log.info("Checking url '%s'", url) + driver.get(self.url) + content = driver.page_source + if self.expected_content: + log.info("Checking url content matches '%s'", self.expected_content) + if self.expected_content not in content: + raise AssertionError('Page source content failed content match') + if self.expected_regex: + log.info("Checking url content matches regex") + if not self.expected_regex.search(content): + raise AssertionError('Page source content failed regex search') + driver.quit() + log.info("Succeeded with capability '%s' against url '%s'", capability, url) + + def run(self): + start_time = time.time() + for capability in self.args: + self.check_selenium(capability, self.url) + query_time = time.time() - start_time + log.info('Finished checks in {:.2f} secs'.format(query_time)) + + +if __name__ == '__main__': + SeleniumTest().main() From bb11b3f4ab5c534ab1bb818e1bf2dc3664e07a18 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 10:50:32 +0100 Subject: [PATCH 1240/2295] updated selenium_test.py --- selenium_test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 3a4118e43..e05f3d081 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -20,9 +20,15 @@ Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME against a given URL and content (defaults to google.com) +Capabilities to check default to 'FIREFOX' and 'CHROME' +URL defaults to 'google.com' +There is no content / regex check by default + Example: - ./selenium_test.py --host [options] + ./selenium_test.py --host [options] [] + + ./selenium_test.py --host selenium-hub ./selenium_test.py --host selenium-hub FIREFOX CHROME @@ -106,7 +112,9 @@ def process_options(self): if self.get_opt('ssl'): self.protocol = 'https' if not self.args: - self.usage() + # test basic Chrome and Firefox are available + self.args.append('CHROME') + self.args.append('FIREFOX') def check_selenium(self, capability, url): selenium_url = '{protocol}://{host}:{port}/{path}'\ From fae117cd04af47c14a212aabf3ba61037628c131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 11:03:34 +0100 Subject: [PATCH 1241/2295] updated selenium_test.py --- selenium_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index e05f3d081..6c13c35e3 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -55,7 +55,7 @@ try: # pylint: disable=wrong-import-position from harisekhon.utils import log - from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex + from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex, die from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') @@ -130,14 +130,17 @@ def check_selenium(self, capability, url): log.info("Checking url '%s'", url) driver.get(self.url) content = driver.page_source - if self.expected_content: - log.info("Checking url content matches '%s'", self.expected_content) - if self.expected_content not in content: - raise AssertionError('Page source content failed content match') if self.expected_regex: log.info("Checking url content matches regex") if not self.expected_regex.search(content): - raise AssertionError('Page source content failed regex search') + die('ERROR: Page source content failed regex search') + elif self.expected_content: + log.info("Checking url content matches '%s'", self.expected_content) + if self.expected_content not in content: + die('ERROR: Page source content failed content match') + elif '404' in driver.title: + die('ERROR: Page title contains a 404 / error ' + + '(if this is expected, use --content / --regex instead): {}'.format(driver.title)) driver.quit() log.info("Succeeded with capability '%s' against url '%s'", capability, url) From 65ae5c0c64f3a0f0221cf8b4db92d02cf473b4fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 11:07:26 +0100 Subject: [PATCH 1242/2295] updated selenium_test.py --- selenium_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 6c13c35e3..e57984ec7 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -130,6 +130,8 @@ def check_selenium(self, capability, url): log.info("Checking url '%s'", url) driver.get(self.url) content = driver.page_source + title = driver.title + driver.quit() if self.expected_regex: log.info("Checking url content matches regex") if not self.expected_regex.search(content): @@ -138,10 +140,9 @@ def check_selenium(self, capability, url): log.info("Checking url content matches '%s'", self.expected_content) if self.expected_content not in content: die('ERROR: Page source content failed content match') - elif '404' in driver.title: + elif '404' in title: die('ERROR: Page title contains a 404 / error ' + - '(if this is expected, use --content / --regex instead): {}'.format(driver.title)) - driver.quit() + '(if this is expected, use --content / --regex instead): {}'.format(title)) log.info("Succeeded with capability '%s' against url '%s'", capability, url) def run(self): From aea9dae435d399344bb23b36af6e30d05e7f1b5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:16:17 +0100 Subject: [PATCH 1243/2295] updated selenium_test.py --- selenium_test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index e57984ec7..7a2459ca8 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -17,16 +17,16 @@ """ -Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME +Tests a Selenium Hub / Selenoid using the given browsers eg. FIREFOX, CHROME against a given URL and content (defaults to google.com) -Capabilities to check default to 'FIREFOX' and 'CHROME' +Browsers default to 'FIREFOX' and 'CHROME' if not specified URL defaults to 'google.com' There is no content / regex check by default Example: - ./selenium_test.py --host [options] [] + ./selenium_test.py --host [] [] ./selenium_test.py --host selenium-hub @@ -113,21 +113,21 @@ def process_options(self): self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available - self.args.append('CHROME') - self.args.append('FIREFOX') + self.args.append('chrome') + self.args.append('firefox') - def check_selenium(self, capability, url): + def check_selenium(self, browser): selenium_url = '{protocol}://{host}:{port}/{path}'\ .format(protocol=self.protocol, \ host=self.host, \ port=self.port, \ path=self.path) - log.info("Connecting to '%s' with capability '%s'", selenium_url, capability) + log.info("Connecting to '%s' for browser '%s'", selenium_url, browser) driver = webdriver.Remote( command_executor=selenium_url, - desired_capabilities=getattr(DesiredCapabilities, capability) + desired_capabilities=getattr(DesiredCapabilities, browser) ) - log.info("Checking url '%s'", url) + log.info("Checking url '%s'", self.url) driver.get(self.url) content = driver.page_source title = driver.title @@ -143,12 +143,12 @@ def check_selenium(self, capability, url): elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, use --content / --regex instead): {}'.format(title)) - log.info("Succeeded with capability '%s' against url '%s'", capability, url) + log.info("Succeeded with capability '%s' against url '%s'", browser, self.url) def run(self): start_time = time.time() - for capability in self.args: - self.check_selenium(capability, self.url) + for browser in self.args: + self.check_selenium(browser.upper()) query_time = time.time() - start_time log.info('Finished checks in {:.2f} secs'.format(query_time)) From b4485faf00545f1fa504269bce876330ef1d9906 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:17:02 +0100 Subject: [PATCH 1244/2295] updated selenium_test.py --- selenium_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 7a2459ca8..cdb5858ee 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -65,11 +65,11 @@ __version__ = '0.1' -class SeleniumTest(CLI): +class SeleniumHubBrowserTest(CLI): def __init__(self): # Python 2.x - super(SeleniumTest, self).__init__() + super(SeleniumHubBrowserTest, self).__init__() # Python 3.x # super().__init__() self.host = None @@ -86,7 +86,7 @@ def __init__(self): self.verbose_default = 2 def add_options(self): - super(SeleniumTest, self).add_options() + super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) @@ -95,7 +95,7 @@ def add_options(self): self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub') def process_options(self): - super(SeleniumTest, self).process_options() + super(SeleniumHubBrowserTest, self).process_options() self.host = self.get_opt('host') self.port = self.get_opt('port') self.url = self.get_opt('url') @@ -154,4 +154,4 @@ def run(self): if __name__ == '__main__': - SeleniumTest().main() + SeleniumHubBrowserTest().main() From 223273ca6e6af6e188eaf1d0432d57f9b86ff83c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:17:12 +0100 Subject: [PATCH 1245/2295] renamed selenium_test.py to seleniumhub_browser_test.py --- selenium_test.py => seleniumhub_browser_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename selenium_test.py => seleniumhub_browser_test.py (100%) diff --git a/selenium_test.py b/seleniumhub_browser_test.py similarity index 100% rename from selenium_test.py rename to seleniumhub_browser_test.py From 43c7be738142c85d8e2a449eaa0c0e6bbad2b8d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:19:46 +0100 Subject: [PATCH 1246/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 18daac3e3..c0493de4a 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- ```seleniumhub_browser_test.py``` - checks [SeleniumHub](https://www.selenium.dev/documentation) by calling chrome and firefox browsers to check a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From c96abc1d55da10cf4d44943fe9a851a8ca20a512 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:21:53 +0100 Subject: [PATCH 1247/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0493de4a..3b9f145e1 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```seleniumhub_browser_test.py``` - checks [SeleniumHub](https://www.selenium.dev/documentation) by calling chrome and firefox browsers to check a given URL and content/regex match the result +- ```seleniumhub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From 6cc7bdcc23af66c1cb37b4e715df1d35b7c0ba48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:23:16 +0100 Subject: [PATCH 1248/2295] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..40a300101 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ python-ldap==3.2.0 python-snappy==0.5 sasl==0.2.1 sh==1.12.14 +selenium==3.141.0 # pulls in python-KrbV as a dependency which doesn't build on Mac any more # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort From 36b03cb3ca0d99032f9a959036a58a0fa1e07b85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:32:11 +0100 Subject: [PATCH 1249/2295] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index cdb5858ee..585e0be4b 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -26,14 +26,32 @@ Example: - ./selenium_test.py --host [] [] + ./seleniumhub_browser_test.py --host [] [] - ./selenium_test.py --host selenium-hub +Where browsers are one or more of these and must be supported by the remote Selenium Hub: - ./selenium_test.py --host selenium-hub FIREFOX CHROME +ANDROID +CHROME +EDGE +FIREFOX +HTMLUNIT +HTMLUNITWITHJS +INTERNETEXPLORER +IPAD +IPHONE +OPERA +PHANTOMJS +SAFARI +WEBKITGTK - ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --content google - ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --regex 'goog.*' +Examples: + + ./seleniumhub_browser_test.py --host x.x.x.x + + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME + + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' """ From 4085be0e61824ad61c6318bba575b18dea3a622a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:11:24 +0100 Subject: [PATCH 1250/2295] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 585e0be4b..905c0b50d 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -80,7 +80,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2' class SeleniumHubBrowserTest(CLI): @@ -127,7 +127,7 @@ def process_options(self): validate_host(self.host) validate_port(self.port) validate_url(self.url) - if self.get_opt('ssl'): + if self.get_opt('ssl') or int(self.port) == 443: self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available From b5548bb26f415df8bb6c7745c03a1342327d4f70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:13:33 +0100 Subject: [PATCH 1251/2295] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 905c0b50d..3a3230925 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -99,6 +99,7 @@ def __init__(self): self.url_default = 'http://google.com' self.url = self.url_default self.expected_content = None + self.expected_content_default = 'google' self.expected_regex = None self.timeout_default = 600 self.verbose_default = 2 @@ -133,6 +134,8 @@ def process_options(self): # test basic Chrome and Firefox are available self.args.append('chrome') self.args.append('firefox') + if self.url == self.url_default: + self.expected_content = self.expected_content_default def check_selenium(self, browser): selenium_url = '{protocol}://{host}:{port}/{path}'\ From ee941e925db2c190a1109a0320f8aaf80ee0316e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:14:18 +0100 Subject: [PATCH 1252/2295] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 3a3230925..1ae8920ed 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -21,8 +21,8 @@ against a given URL and content (defaults to google.com) Browsers default to 'FIREFOX' and 'CHROME' if not specified -URL defaults to 'google.com' -There is no content / regex check by default +URL defaults to 'google.com' checking for content 'google' +If you define a different URL then you must specify a --content or --regex validation otherwise none is used Example: From 3f257d161ba330cdd984cc8d51e187c71fcdb212 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:48:17 +0100 Subject: [PATCH 1253/2295] added --hub-url option --- seleniumhub_browser_test.py | 46 ++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 1ae8920ed..84456583d 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -26,7 +26,9 @@ Example: - ./seleniumhub_browser_test.py --host [] [] + ./selenium_hub_browser_test.py --host [] [] + + ./selenium_hub_browser_test.py --hub-url https://:4444/wd/hub/ [] [] Where browsers are one or more of these and must be supported by the remote Selenium Hub: @@ -46,12 +48,12 @@ Examples: - ./seleniumhub_browser_test.py --host x.x.x.x + ./selenium_hub_browser_test.py --host x.x.x.x - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' """ @@ -80,7 +82,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2' +__version__ = '0.3' class SeleniumHubBrowserTest(CLI): @@ -96,6 +98,7 @@ def __init__(self): self.name = 'Selenium Hub' self.default_port = 80 self.path = 'wd/hub' + self.hub_url = None self.url_default = 'http://google.com' self.url = self.url_default self.expected_content = None @@ -107,6 +110,7 @@ def __init__(self): def add_options(self): super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) + self.add_opt('-U', '--hub-url', help='Selenium Hub URL') self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) self.add_opt('-c', '--content', help='URL content to expect') @@ -115,8 +119,21 @@ def add_options(self): def process_options(self): super(SeleniumHubBrowserTest, self).process_options() - self.host = self.get_opt('host') - self.port = self.get_opt('port') + self.hub_url = self.get_opt('hub_url') + if self.hub_url: + validate_url(self.hub_url, 'hub') + else: + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + if self.get_opt('ssl') or int(self.port) == 443: + self.protocol = 'https' + self.hub_url = '{protocol}://{host}:{port}/{path}'\ + .format(protocol=self.protocol, \ + host=self.host, \ + port=self.port, \ + path=self.path) self.url = self.get_opt('url') if ':' not in self.url: self.url = 'http://' + self.url @@ -125,11 +142,7 @@ def process_options(self): if self.expected_regex: validate_regex(self.expected_regex) self.expected_regex = re.compile(self.expected_regex) - validate_host(self.host) - validate_port(self.port) validate_url(self.url) - if self.get_opt('ssl') or int(self.port) == 443: - self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available self.args.append('chrome') @@ -138,14 +151,9 @@ def process_options(self): self.expected_content = self.expected_content_default def check_selenium(self, browser): - selenium_url = '{protocol}://{host}:{port}/{path}'\ - .format(protocol=self.protocol, \ - host=self.host, \ - port=self.port, \ - path=self.path) - log.info("Connecting to '%s' for browser '%s'", selenium_url, browser) + log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser) driver = webdriver.Remote( - command_executor=selenium_url, + command_executor=self.hub_url, desired_capabilities=getattr(DesiredCapabilities, browser) ) log.info("Checking url '%s'", self.url) From dc11c44e5b8935dd3b95f5fa82181b2ff265c5b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:48:24 +0100 Subject: [PATCH 1254/2295] renamed seleniumhub_browser_test.py to selenium_hub_browser_test.py --- seleniumhub_browser_test.py => selenium_hub_browser_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename seleniumhub_browser_test.py => selenium_hub_browser_test.py (100%) diff --git a/seleniumhub_browser_test.py b/selenium_hub_browser_test.py similarity index 100% rename from seleniumhub_browser_test.py rename to selenium_hub_browser_test.py From e87d99685efa3a26523c4391b17de2e0ad7b7c79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:50:54 +0100 Subject: [PATCH 1255/2295] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40a300101..a64f54236 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 -Jinja2==2.10.1 +jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 From b1fa126f6c0044c05bd30aa1433409e18306d5e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:52:05 +0100 Subject: [PATCH 1256/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b9f145e1..6cb10d2f7 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```seleniumhub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result +- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From 583296e3dc706f94ccd6273f1817106e9d4a4078 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:57:43 +0100 Subject: [PATCH 1257/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 84456583d..7ea5f1860 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -96,7 +96,6 @@ def __init__(self): self.port = None self.protocol = 'http' self.name = 'Selenium Hub' - self.default_port = 80 self.path = 'wd/hub' self.hub_url = None self.url_default = 'http://google.com' @@ -110,7 +109,7 @@ def __init__(self): def add_options(self): super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) - self.add_opt('-U', '--hub-url', help='Selenium Hub URL') + self.add_opt('-U', '--hub-url', help='Selenium Hub URL (takes priority over --host/--port/--ssl)') self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) self.add_opt('-c', '--content', help='URL content to expect') From aacd839656f8c7c6435cbadc885b86df5caf6f1b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:02:18 +0100 Subject: [PATCH 1258/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 7ea5f1860..462973872 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -136,18 +136,18 @@ def process_options(self): self.url = self.get_opt('url') if ':' not in self.url: self.url = 'http://' + self.url + validate_url(self.url) self.expected_content = self.get_opt('content') self.expected_regex = self.get_opt('regex') if self.expected_regex: validate_regex(self.expected_regex) self.expected_regex = re.compile(self.expected_regex) - validate_url(self.url) + elif self.url == self.url_default: + self.expected_content = self.expected_content_default if not self.args: # test basic Chrome and Firefox are available self.args.append('chrome') self.args.append('firefox') - if self.url == self.url_default: - self.expected_content = self.expected_content_default def check_selenium(self, browser): log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser) From 90dbc0823dbfe9f3c96f5cc4375ff4faa9d5d5cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:05:42 +0100 Subject: [PATCH 1259/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 462973872..7a5472374 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -74,9 +74,8 @@ sys.path.append(libdir) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex, die from harisekhon import CLI + from harisekhon.utils import log, validate_host, validate_port, validate_url, validate_regex, die except ImportError as _: print(traceback.format_exc(), end='') sys.exit(4) From b09f912723fcf1144955c032831924aca3ed251d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:12:01 +0100 Subject: [PATCH 1260/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 7a5472374..00734c969 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -170,7 +170,7 @@ def check_selenium(self, browser): elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, use --content / --regex instead): {}'.format(title)) - log.info("Succeeded with capability '%s' against url '%s'", browser, self.url) + log.info("Succeeded for browser '%s' against url '%s'", browser, self.url) def run(self): start_time = time.time() From 8ecff7acdd9bb0f335ca31deddbbf6f6bb9a303c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:23:45 +0100 Subject: [PATCH 1261/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 00734c969..8d498cc9a 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -169,7 +169,7 @@ def check_selenium(self, browser): die('ERROR: Page source content failed content match') elif '404' in title: die('ERROR: Page title contains a 404 / error ' + - '(if this is expected, use --content / --regex instead): {}'.format(title)) + '(if this is expected, specify --content / --regex to check instead): {}'.format(title)) log.info("Succeeded for browser '%s' against url '%s'", browser, self.url) def run(self): From 491e8723da7b61d45f7366ae958ab23673bee1fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:31:14 +0100 Subject: [PATCH 1262/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 8d498cc9a..f67c45327 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -28,7 +28,7 @@ ./selenium_hub_browser_test.py --host [] [] - ./selenium_hub_browser_test.py --hub-url https://:4444/wd/hub/ [] [] + ./selenium_hub_browser_test.py --hub-url http://:4444/wd/hub/ [] [] Where browsers are one or more of these and must be supported by the remote Selenium Hub: @@ -55,6 +55,8 @@ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' + +Tested on Selenium Grid Hub v.3.141.59, and Selenoid 1.10.1 """ from __future__ import absolute_import From 37699e74f1b787268cdd3ba076c3adbe090b14c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:34 +0100 Subject: [PATCH 1263/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7b69c6157..528aa8999 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7b69c61571a3a27b786884999fd0937a8150cc78 +Subproject commit 528aa89996f39b5d6c4d0df41404d9ca4c6fff3d From 717dfde4c3e1593f5f79cb1cc17f3df54cf8d78e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:34 +0100 Subject: [PATCH 1264/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 65631842d..964ff75d9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 65631842daa333da9e8f5ca4ea26f30d380e53b6 +Subproject commit 964ff75d90eb7337a998bbab4e832ca7307b4f04 From 5723131b1e4865e0eab85e59461352be653cb55b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:35 +0100 Subject: [PATCH 1265/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a9ebed85a..4775e138b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a9ebed85a6753ef025b8d1c49fdb6a466bb66727 +Subproject commit 4775e138bbb851f9157e318060f420389d3b695d From 9a26ae3df818bb9ecb03bac0aa58164c82214012 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:14:32 +0100 Subject: [PATCH 1266/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f67c45327..f9fd17824 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -169,6 +169,10 @@ def check_selenium(self, browser): log.info("Checking url content matches '%s'", self.expected_content) if self.expected_content not in content: die('ERROR: Page source content failed content match') + # not really recommended but in this case we cannot predict + # what to expect on a random url if not specified by --content/--regex: + # + # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/ elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, specify --content / --regex to check instead): {}'.format(title)) From 198c10ae442105e2ae16c58dbcd7c090787d4abe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:22:32 +0100 Subject: [PATCH 1267/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f9fd17824..cb8bebd99 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -170,7 +170,7 @@ def check_selenium(self, browser): if self.expected_content not in content: die('ERROR: Page source content failed content match') # not really recommended but in this case we cannot predict - # what to expect on a random url if not specified by --content/--regex: + # what to expect on a random url if not specified by --content/--regex (provided in the default test case) # # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/ elif '404' in title: From 5d3dc09060b7336ad2536cf6ab63d42380e502ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:23:12 +0100 Subject: [PATCH 1268/2295] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aedc339bf..dfef9fec9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,4 +57,4 @@ thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 -pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability +#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found From cdc3e04c66329659930df52b2f78712afd06cc87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 May 2021 11:06:34 +0100 Subject: [PATCH 1269/2295] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index cb8bebd99..8b0242b83 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -56,7 +56,7 @@ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' -Tested on Selenium Grid Hub v.3.141.59, and Selenoid 1.10.1 +Tested on Selenium Grid Hub v.3.141.59, v4.0.0 and Selenoid 1.10.1 """ from __future__ import absolute_import From 2683ba9b7564b178a589e76cd3f7bc1cae940ed9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:10:10 +0100 Subject: [PATCH 1270/2295] added main.py --- gcp_cloud_function_ifconfig/main.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 gcp_cloud_function_ifconfig/main.py diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py new file mode 100755 index 000000000..ba3089e20 --- /dev/null +++ b/gcp_cloud_function_ifconfig/main.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +The HTTP request is irrelevant, although we could pass the website to query via an arg in which case this would just act as a proxy + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import requests + +def main(request): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + r = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing + status_code = r.status_code + status_message = r.reason + content = r.text + return "{} {}\n\n{}".format(status_code, status_message, content) From 5bacef94a5296245b440c98975fa8d9cb7bdfed2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:15:00 +0100 Subject: [PATCH 1271/2295] updated main.py --- gcp_cloud_function_ifconfig/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py index ba3089e20..a2f4e6783 100755 --- a/gcp_cloud_function_ifconfig/main.py +++ b/gcp_cloud_function_ifconfig/main.py @@ -22,7 +22,8 @@ Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules -The HTTP request is irrelevant, although we could pass the website to query via an arg in which case this would just act as a proxy +The HTTP request is irrelevant, although we could pass the website to query via an arg +in which case this would just act as a proxy. See the adjacent gcp_cloud_function_proxy/main.py Tested on GCP Cloud Functions with Python 3.9 @@ -32,7 +33,7 @@ import requests -def main(request): +def main(_): """Responds to any HTTP request. Args: request (flask.Request): HTTP request object. @@ -41,8 +42,8 @@ def main(request): Response object using `make_response `. """ - r = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing - status_code = r.status_code - status_message = r.reason - content = r.text + req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing + status_code = req.status_code + status_message = req.reason + content = req.text return "{} {}\n\n{}".format(status_code, status_message, content) From f26a380d667ad5bd4490ef422521e9dfadfc8007 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:26:56 +0100 Subject: [PATCH 1272/2295] added main.py --- gcp_cloud_function_proxy/main.py | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 gcp_cloud_function_proxy/main.py diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py new file mode 100755 index 000000000..3c6209ca6 --- /dev/null +++ b/gcp_cloud_function_proxy/main.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: + +Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +Test request examples: + + { "url": "http://ifconfig.co/json" } + +defaults to http:// if not specified: + + { "url": "ifconfig.co/json" } + + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import json +import requests + +def main(request): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + data = json.loads(request.data) + url = data['url'] + if '://' not in url: + url = 'http://' + url + req = requests.get(url) + status_code = req.status_code + status_message = req.reason + content = req.text + return "{} {}\n\n{}".format(status_code, status_message, content) From e8f457b5eac3d165f0b3d15aa2881e543337a155 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:30:29 +0100 Subject: [PATCH 1273/2295] updated main.py --- gcp_cloud_function_ifconfig/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py index a2f4e6783..309bc58c0 100755 --- a/gcp_cloud_function_ifconfig/main.py +++ b/gcp_cloud_function_ifconfig/main.py @@ -22,13 +22,18 @@ Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules -The HTTP request is irrelevant, although we could pass the website to query via an arg -in which case this would just act as a proxy. See the adjacent gcp_cloud_function_proxy/main.py +The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query +in a field in which case this would just act as a proxy. + +See Also: gcp_cloud_function_proxy/main.py + Tested on GCP Cloud Functions with Python 3.9 """ +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python import requests From d739b4e24301ec0c278b638cda34e0158b70d66a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:31:14 +0100 Subject: [PATCH 1274/2295] updated main.py --- gcp_cloud_function_proxy/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py index 3c6209ca6..9e525e72b 100755 --- a/gcp_cloud_function_proxy/main.py +++ b/gcp_cloud_function_proxy/main.py @@ -37,6 +37,8 @@ """ +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python import json From 12e626c93b05ea461ecd3e0325a670661304154a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:32:27 +0100 Subject: [PATCH 1275/2295] added requirements.txt --- gcp_cloud_function_proxy/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 gcp_cloud_function_proxy/requirements.txt diff --git a/gcp_cloud_function_proxy/requirements.txt b/gcp_cloud_function_proxy/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_proxy/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 From 914f86d3dfabaf5b33b1b544a5db405a89896b2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:32:42 +0100 Subject: [PATCH 1276/2295] added requirements.txt --- gcp_cloud_function_ifconfig/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/requirements.txt diff --git a/gcp_cloud_function_ifconfig/requirements.txt b/gcp_cloud_function_ifconfig/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_ifconfig/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 From e68719a6d74ff5f868c57f800b8cd0c8bec51bc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:33:24 +0100 Subject: [PATCH 1277/2295] added .gcloudignore --- gcp_cloud_function_proxy/.gcloudignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_proxy/.gcloudignore diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore new file mode 100644 index 000000000..8f605abc2 --- /dev/null +++ b/gcp_cloud_function_proxy/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ From 9ba9d0ba5e610ee1fc019b2fa08469dfb299814b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:33:35 +0100 Subject: [PATCH 1278/2295] added .gcloudignore --- gcp_cloud_function_ifconfig/.gcloudignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/.gcloudignore diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore new file mode 100644 index 000000000..8f605abc2 --- /dev/null +++ b/gcp_cloud_function_ifconfig/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ From de6f67df8583ecfc2c3354acd461897a10689323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:43:59 +0100 Subject: [PATCH 1279/2295] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 1465dbc56..62857a9c7 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -19,9 +19,11 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir" -project="$(gcloud config list --format="value(core.project)")" -region="$(gcloud config list --format="value(compute.region)")" -region="${region:-${GOOGLE_REGION:-europe-west1}}" # not available in all regions yet +# needed to define the $service_account further down +project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" + +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet name="cloud-sql-backups" topic="cloud-sql-backups" From 43eaa6fc890a70bded7e86fcf9ecddb618a9aa07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:45:26 +0100 Subject: [PATCH 1280/2295] added deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 gcp_cloud_function_ifconfig/deploy.sh diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh new file mode 100755 index 000000000..c103f9efe --- /dev/null +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="ifconfig" + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="ifconfig" + +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --timeout 60 \ + --quiet # don't prompt to --allow-unauthenticated + #--max-instances 1 + #--vpc-connector "$vpc_connector" From 004d3b13ad4c6c6f03ac33f7e31b948627fcbfd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:59:51 +0100 Subject: [PATCH 1281/2295] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 62857a9c7..64705ad5c 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -22,6 +22,7 @@ cd "$srcdir" # needed to define the $service_account further down project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet From 6c25b875d833c6922a1ad220dced49a41ba93721 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:02:59 +0100 Subject: [PATCH 1282/2295] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index c103f9efe..aa1fbe03b 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -21,17 +21,22 @@ cd "$srcdir" name="ifconfig" +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet + # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs # since we're only accessing the SQL Admin API we don't need this -#vpc_connector="ifconfig" +#vpc_connector="my-vpc-connector" gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ --entry-point main \ --memory 128MB \ + --region "$region" \ --timeout 60 \ --quiet # don't prompt to --allow-unauthenticated - #--max-instances 1 - #--vpc-connector "$vpc_connector" + #--vpc-connector "$vpc_connector" \ + #--max-instances 1 \ From db8f71b08872c8dec774e6fd9284a35a7446f9b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:08:50 +0100 Subject: [PATCH 1283/2295] added deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 gcp_cloud_function_proxy/deploy.sh diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh new file mode 100755 index 000000000..d77f41c04 --- /dev/null +++ b/gcp_cloud_function_proxy/deploy.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="proxy" + +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="my-vpc-connector" + +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --region "$region" \ + --timeout 60 \ + --quiet # don't prompt to --allow-unauthenticated + # routes private traffic only by default - need to change this if you want to + # use the Cloud NAT gateway from the VPC for Firewall rules purposes + #--vpc-connector "$vpc_connector" \ + #--max-instances 1 From 8ace6d77ec56a353f8284e4ff3babca347cdd04f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:11:03 +0100 Subject: [PATCH 1284/2295] added README.md --- gcp_cloud_function_ifconfig/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/README.md diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md new file mode 100644 index 000000000..d75717529 --- /dev/null +++ b/gcp_cloud_function_ifconfig/README.md @@ -0,0 +1,14 @@ +Google Cloud Function - SQL Backup Exporter to GCS +===================== + +Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` From 6e27129b2fb7ab6dbd74e5c38f845b9184656819 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:12:22 +0100 Subject: [PATCH 1285/2295] added README.md --- gcp_cloud_function_proxy/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 gcp_cloud_function_proxy/README.md diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md new file mode 100644 index 000000000..c0368b1c2 --- /dev/null +++ b/gcp_cloud_function_proxy/README.md @@ -0,0 +1,19 @@ +Google Cloud Function - SQL Backup Exporter to GCS +===================== + +Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules + +The query string should be like so: +``` +{"url": "http://ifconfig.co/json"} +``` + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` From 9bf8e665e455df4c5021f43abc4a067d8f4bb235 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:32:54 +0100 Subject: [PATCH 1286/2295] updated README.md --- gcp_cloud_function_proxy/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index c0368b1c2..78c6ff1f1 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -1,13 +1,22 @@ -Google Cloud Function - SQL Backup Exporter to GCS +Google Cloud Function - Proxy ===================== Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules -The query string should be like so: +Query content: ``` {"url": "http://ifconfig.co/json"} ``` +Response: + +``` +200 OK + + +``` + + - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps From b6118c024e33768845dc031a7bb0e4a23e442fac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:33:36 +0100 Subject: [PATCH 1287/2295] updated README.md --- gcp_cloud_function_proxy/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index 78c6ff1f1..9cee20eb8 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -11,11 +11,20 @@ Query content: Response: ``` -200 OK + ``` +eg. + +``` +200 OK + + +... +``` + - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap From 29829f1c0e512f9206b2b7dd5da71860e6e8ba3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:34:49 +0100 Subject: [PATCH 1288/2295] updated README.md --- gcp_cloud_function_proxy/README.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index 9cee20eb8..1a4c2465a 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -8,24 +8,14 @@ Query content: {"url": "http://ifconfig.co/json"} ``` -Response: - -``` - - - -``` - -eg. +Response is HTTP status code and message, blank line and then the content: ``` 200 OK - -... + ``` - - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps From 59750ea035f53f7af608ae13d16759ae654c5d9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:37:53 +0100 Subject: [PATCH 1289/2295] added Makefile --- gcp_cloud_function_sql_export/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_sql_export/Makefile diff --git a/gcp_cloud_function_sql_export/Makefile b/gcp_cloud_function_sql_export/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_sql_export/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From d196c0e4b48b617b88e07f3c8e7f8b77dc170f94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:38:08 +0100 Subject: [PATCH 1290/2295] added Makefile --- gcp_cloud_function_ifconfig/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/Makefile diff --git a/gcp_cloud_function_ifconfig/Makefile b/gcp_cloud_function_ifconfig/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_ifconfig/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From 88a406f911885efa2fedeba388e45b215a036e10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:38:17 +0100 Subject: [PATCH 1291/2295] added Makefile --- gcp_cloud_function_proxy/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_proxy/Makefile diff --git a/gcp_cloud_function_proxy/Makefile b/gcp_cloud_function_proxy/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_proxy/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From 68d0b7ad41d083b5e476008c29a3ffea0ea59109 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:13:45 +0100 Subject: [PATCH 1292/2295] updated README.md --- gcp_cloud_function_ifconfig/README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md index d75717529..be6e431a1 100644 --- a/gcp_cloud_function_ifconfig/README.md +++ b/gcp_cloud_function_ifconfig/README.md @@ -1,4 +1,4 @@ -Google Cloud Function - SQL Backup Exporter to GCS +Google Cloud Function - ifconfig ===================== Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules @@ -7,6 +7,31 @@ Queries http://ifconfig.co from GCF to check the routing and external IP being u - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps +Response is HTTP status code and message, then the raw JSON results + +``` +200 OK + +{ + "ip": "...", + "ip_decimal": ... , + "country": "United States", + "country_iso": "US", + "country_eu": false, + "latitude": 37.751, + "longitude": -97.822, + "time_zone": "America/Chicago", + "asn": "AS15169", + "asn_org": "GOOGLE", + "hostname": "ipv6.gae.googleusercontent.com", + "user_agent": { + "product": "python-requests", + "version": "2.24.0", + "raw_value": "python-requests/2.24.0" + } +} +``` + Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: ``` From abead35e94e5e3a54ef456dd9a4fc5974e9e2214 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:15:56 +0100 Subject: [PATCH 1293/2295] updated README.md --- gcp_cloud_function_ifconfig/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md index be6e431a1..0485fcf08 100644 --- a/gcp_cloud_function_ifconfig/README.md +++ b/gcp_cloud_function_ifconfig/README.md @@ -13,8 +13,8 @@ Response is HTTP status code and message, then the raw JSON results 200 OK { - "ip": "...", - "ip_decimal": ... , + "ip": "1.2.3.4", + "ip_decimal": 1234567890, "country": "United States", "country_iso": "US", "country_eu": false, @@ -32,6 +32,12 @@ Response is HTTP status code and message, then the raw JSON results } ``` +For IPv6 the format will be more like: +``` + "ip": "1234:5678:9012:34::a", + "ip_decimal": 12345678901234567890123456789012345678, +``` + Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: ``` From fccb4e40c3c70d7cb2ceb2970840551a5221d4a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:20:27 +0100 Subject: [PATCH 1294/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6cb10d2f7..2a48b68fb 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,8 @@ Environment variables are supported for convenience and also to hide credentials - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function public networking by determining which IP address your cloud function is seen as coming from + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function networking by querying a given URL to check accessibility, returning the HTTP status code and content - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 6f28bdba976126c2b973aece6c7e99e4fad3831a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:23:49 +0100 Subject: [PATCH 1295/2295] updated README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2a48b68fb..467cb7c86 100644 --- a/README.md +++ b/README.md @@ -188,10 +188,11 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function public networking by determining which IP address your cloud function is seen as coming from - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function networking by querying a given URL to check accessibility, returning the HTTP status code and content + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From ef9be89b6f36f0689b16d72f45de530415c1482e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:23:58 +0100 Subject: [PATCH 1296/2295] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 467cb7c86..d36cef7cc 100644 --- a/README.md +++ b/README.md @@ -189,10 +189,10 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 979acb46c157840ad63779076d13218196acf15b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:05 +0100 Subject: [PATCH 1297/2295] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index aa1fbe03b..015234fb7 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -21,9 +21,13 @@ cd "$srcdir" name="ifconfig" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs From 432d2a79eb04a065ebc11a87d642d308c9411131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:19 +0100 Subject: [PATCH 1298/2295] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index d77f41c04..da82e737c 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -21,9 +21,13 @@ cd "$srcdir" name="proxy" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs From 5afaf5ffa7403c6f19435f213dadd2dda97c52e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:48 +0100 Subject: [PATCH 1299/2295] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 64705ad5c..3dd2c44f5 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -22,9 +22,13 @@ cd "$srcdir" # needed to define the $service_account further down project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" name="cloud-sql-backups" topic="cloud-sql-backups" From d4bb08e84801dc3a664137eb553d6ce8de90520f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 17:31:19 +0100 Subject: [PATCH 1300/2295] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index da82e737c..9854672cd 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -45,4 +45,5 @@ gcloud functions deploy "$name" --trigger-http \ # routes private traffic only by default - need to change this if you want to # use the Cloud NAT gateway from the VPC for Firewall rules purposes #--vpc-connector "$vpc_connector" \ + #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 From 174a6f2867c5ea027a6436b45a7e9db86fdf513c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 17:31:30 +0100 Subject: [PATCH 1301/2295] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index 015234fb7..f24a04a45 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -42,5 +42,8 @@ gcloud functions deploy "$name" --trigger-http \ --region "$region" \ --timeout 60 \ --quiet # don't prompt to --allow-unauthenticated + # routes private traffic only by default - need to change this if you want to + # use the Cloud NAT gateway from the VPC for Firewall rules purposes #--vpc-connector "$vpc_connector" \ + #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 \ From 131bf5caabb4a223a3fd2090f6201639763bc8eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 18:45:20 +0100 Subject: [PATCH 1302/2295] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index f24a04a45..6f7fc0ab3 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -34,6 +34,13 @@ region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # since we're only accessing the SQL Admin API we don't need this #vpc_connector="my-vpc-connector" +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ @@ -41,9 +48,6 @@ gcloud functions deploy "$name" --trigger-http \ --memory 128MB \ --region "$region" \ --timeout 60 \ + "${opts[@]}" \ --quiet # don't prompt to --allow-unauthenticated - # routes private traffic only by default - need to change this if you want to - # use the Cloud NAT gateway from the VPC for Firewall rules purposes - #--vpc-connector "$vpc_connector" \ - #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 \ From d70585017e42de536a7053b991e1e6fafdd9bce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 19:00:00 +0100 Subject: [PATCH 1303/2295] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index 9854672cd..a583c1d2e 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -34,6 +34,13 @@ region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # since we're only accessing the SQL Admin API we don't need this #vpc_connector="my-vpc-connector" +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ @@ -41,9 +48,6 @@ gcloud functions deploy "$name" --trigger-http \ --memory 128MB \ --region "$region" \ --timeout 60 \ + "${opts[@]}" \ --quiet # don't prompt to --allow-unauthenticated - # routes private traffic only by default - need to change this if you want to - # use the Cloud NAT gateway from the VPC for Firewall rules purposes - #--vpc-connector "$vpc_connector" \ - #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 From 372749efe55533903a72c51a5452a78934cf7530 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:29:19 +0100 Subject: [PATCH 1304/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d36cef7cc..62a9153c7 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. -* [Kubernetes templates](https://github.com/HariSekhon/Kubernetes-templates) - Kubernetes YAML templates - Best Practices, Tips & Tricks are baked right into the templates for future deployments +* [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) From ab3adcfe269c8c3c16ad5ebb7dcac14b8243fe82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:20 +0100 Subject: [PATCH 1305/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 528aa8999..38f6feaef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 528aa89996f39b5d6c4d0df41404d9ca4c6fff3d +Subproject commit 38f6feaef03246160bf107fdf82ac84962b0555c From 79fc0e51188181498f567571a20f94abcb35bda9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:20 +0100 Subject: [PATCH 1306/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 964ff75d9..bbffddce9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 964ff75d90eb7337a998bbab4e832ca7307b4f04 +Subproject commit bbffddce93a5d9a934e4fa3ae21af2008e43061d From 538157988744146fa7545555753b213ed02a8cf7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:21 +0100 Subject: [PATCH 1307/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 80e2004a1..bdd9588ea 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 80e2004a17865c23992f3ad8d5bae3d55e60dc46 +Subproject commit bdd9588ea844c843cfaf2c054e485b85fe85bd82 From 5b992595ddc40240ddc2b47aaa771c3cfb6cee0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:21 +0100 Subject: [PATCH 1308/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 4775e138b..cd774ac45 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 4775e138bbb851f9157e318060f420389d3b695d +Subproject commit cd774ac45526f9fa0263136ffbf5176db7b0a756 From d40ee46f35922d334e1263ca9851337a841502d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:04 +0100 Subject: [PATCH 1309/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 38f6feaef..9bfa74658 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 38f6feaef03246160bf107fdf82ac84962b0555c +Subproject commit 9bfa74658bfd271e0912306e053ec5690fa834e1 From 99f1f60ead060c78ed80e18a73520c71f656802a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1310/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bbffddce9..70bd42751 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bbffddce93a5d9a934e4fa3ae21af2008e43061d +Subproject commit 70bd427512cd2f862cf7b9622e8fa32b11726a75 From 424bbc81afe68c58789baf31aa9d74accdf246f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1311/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index bdd9588ea..b9c39d4e4 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit bdd9588ea844c843cfaf2c054e485b85fe85bd82 +Subproject commit b9c39d4e4e121a9f83f3a588df8bd17f94f57dde From 5e304641be41f88d527ac4c16e876572d81b90dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1312/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index cd774ac45..994300130 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit cd774ac45526f9fa0263136ffbf5176db7b0a756 +Subproject commit 9943001302a2751781bf345584d100e6afcb5b07 From 77bd0b858d93b39d9bef1f24c3f35f0b26d0fdb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 17:37:22 +0100 Subject: [PATCH 1313/2295] updated Makefile --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) mode change 100755 => 100644 Makefile diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 index abe77d452..66b95292f --- a/Makefile +++ b/Makefile @@ -78,8 +78,14 @@ init: # # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # +# doesn't work +#.PHONY: all +#.PHONY: %.pyc +# TODO: doesn't work, says nothing to be done even when .pyc isn't present, and allows make anonymize22.py which breaks +#%.py: %.pyc +# @$(MAKE) $@c %.pyc:: %.py - @# this utility script supports taking .pyc or .pyo names and still does the right thing, + @# this utility script supports taking .pyc or .pyo names and still does the right thing @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ From cb6d31015052ea361bab2e4b19c3bcb56ac95966 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jul 2021 12:32:08 +0100 Subject: [PATCH 1314/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62a9153c7..08255495b 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis%20CI)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) From 4253835d19bfe7ece79333793bbb8750fd606b7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:17 +0100 Subject: [PATCH 1315/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9bfa74658..743fda601 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9bfa74658bfd271e0912306e053ec5690fa834e1 +Subproject commit 743fda60144424b8926fbd1851d5f99209c09cc0 From cfd9e884a288da2e1b70155a99f1f34fc97a2db8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1316/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 70bd42751..2c756632a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 70bd427512cd2f862cf7b9622e8fa32b11726a75 +Subproject commit 2c756632a83aad4b456f8f86d90de40f3c6622e4 From 40666b12c3fe7c618331ea13cbbf5fe43903c47e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1317/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b9c39d4e4..25e563ad7 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b9c39d4e4e121a9f83f3a588df8bd17f94f57dde +Subproject commit 25e563ad72735fe868ed01534b3313322b68f49e From 61977ccfb5869b6c4691918930650da70cac0181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1318/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 994300130..9c0c67b07 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9943001302a2751781bf345584d100e6afcb5b07 +Subproject commit 9c0c67b07d55aca66986d0d2b62c7952e72f6afc From 8e8f7d10de266d884e66a46978b7f4cd860c4c2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:34:24 +0100 Subject: [PATCH 1319/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 09b556190..6e5dc9f25 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -67,7 +67,7 @@ blocks: agent: machine: type: a1-standard-4 - os_image: macos-xcode11 + os_image: macos-xcode12 prologue: commands: - cache restore From 6e4f74da5f97aabbb2f32ba4a0621fa128f1833c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1320/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 743fda601..14feb8065 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 743fda60144424b8926fbd1851d5f99209c09cc0 +Subproject commit 14feb80653901b7f59a4fc3b5ea8ca57d1e25eda From b36a00091c04c1155f112ad7fab580489a1c7fe4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1321/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2c756632a..bb608550d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2c756632a83aad4b456f8f86d90de40f3c6622e4 +Subproject commit bb608550d7163fde875e0df9e31bacbfa77350e3 From 1dcfe0cd4be79b8088cc6f8395bd0e413163ca37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1322/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 25e563ad7..d1989db29 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 25e563ad72735fe868ed01534b3313322b68f49e +Subproject commit d1989db297219f5fa0fa81f213a3a098df01f9fb From e386857bb0b97788938aeb8d741d287378557852 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1323/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9c0c67b07..c0a4f0054 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9c0c67b07d55aca66986d0d2b62c7952e72f6afc +Subproject commit c0a4f005460f04e104baa1c0d8095e4fdc687696 From 93dcd18b057d9dd93321d6c36ba456479626389d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Oct 2021 16:37:58 +0100 Subject: [PATCH 1324/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14feb8065..248c4b862 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14feb80653901b7f59a4fc3b5ea8ca57d1e25eda +Subproject commit 248c4b86270c680a38a70fc462bd35ff06dfbdb6 From 8010521b0399950ffe490acfa8cd6f545b989063 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Nov 2021 19:08:51 +0000 Subject: [PATCH 1325/2295] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..d6aee201a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '37 15 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From a9678fbfbefcf34f575a5be2c997d441b925f21d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 19 Nov 2021 16:47:38 +0000 Subject: [PATCH 1326/2295] stripped quay.io from start if found for easier pasting --- quay_show_tags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/quay_show_tags.py b/quay_show_tags.py index 5743be955..64a2167e6 100755 --- a/quay_show_tags.py +++ b/quay_show_tags.py @@ -30,6 +30,7 @@ #from __future__ import unicode_literals import os +import re import sys import traceback srcdir = os.path.abspath(os.path.dirname(__file__)) @@ -43,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class QuayTags(DockerHubTags): @@ -61,7 +62,10 @@ def run(self): self.quiet = self.get_opt('quiet') if not self.quiet: print('\nQuay.io ', end='') + re_quay = re.compile('^quay.io/', re.I) for arg in self.args: + if re_quay.match(arg): + arg = re_quay.sub('', arg) self.print_tags(arg) From fbdaf03607876b6c547bca0386137ccf3998f67c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 18:41:32 +0000 Subject: [PATCH 1327/2295] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 08255495b..68a99b2d8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ Hari Sekhon - DevOps Python Tools [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) +[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 54d9a8388367bdd2ccf444a784b2f098d044558e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 18:47:57 +0000 Subject: [PATCH 1328/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68a99b2d8..2c9968f30 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) [![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) From c13fb459ea57eb9080d3ffe2e40ce265835f062e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:15:35 +0000 Subject: [PATCH 1329/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c9968f30..d9150f343 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ All programs come with a ```--help``` switch which includes a program descriptio Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. -### DevOps Python Tools - Inventory: +### DevOps Python Tools - Inventory - Linux: - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) From 5741cb779f688b2b2983fb683803c2ceb4dc73a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:18:20 +0000 Subject: [PATCH 1330/2295] updated README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d9150f343..f0d4b1863 100644 --- a/README.md +++ b/README.md @@ -463,33 +463,33 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies -* [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery -* [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. +- [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. -* [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments +- [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... -* [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. +- [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. -* [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 +- [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 -* [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +- [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule -* [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +- [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: -* https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb ### Stargazers over time From c3e3b8c4dd03835fb73136ef925703ecc34e69c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1331/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 248c4b862..9e172499f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 248c4b86270c680a38a70fc462bd35ff06dfbdb6 +Subproject commit 9e172499fad04aff95a06af08750187adbc82293 From 6138988cce450a9b4d1585cfd342390fde3c8f3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1332/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bb608550d..ce9da2c5e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bb608550d7163fde875e0df9e31bacbfa77350e3 +Subproject commit ce9da2c5ed3f1b8f4c275c3349a83055a46d3c41 From 9f1dd7789fe7eb2109cabf6806e493b58e3b69d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1333/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d1989db29..53c1d53b0 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d1989db297219f5fa0fa81f213a3a098df01f9fb +Subproject commit 53c1d53b0ff15f8d9d0b7191b522a1b0b1d0bf20 From f889d4d7ac0fd5f9c29c21393f8aba1968242837 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1334/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c0a4f0054..9021150df 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c0a4f005460f04e104baa1c0d8095e4fdc687696 +Subproject commit 9021150df83050e91ff5c72894de46305d0d54ce From c6831e3d589a98d61910d6e13c7de6d284dd74d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 09:54:42 +0000 Subject: [PATCH 1335/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9e172499f..46e0915c8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9e172499fad04aff95a06af08750187adbc82293 +Subproject commit 46e0915c89142fb4c8a9f79e23d63cba1b76862f From ae5fb27cf2b11e288e0a7ebaed960797f4ce0b66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 11:14:09 +0000 Subject: [PATCH 1336/2295] cache bust trigger test From 179e61581ff21dd644909f1f99045817181a2d6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 11:50:06 +0000 Subject: [PATCH 1337/2295] cache bust trigger test From 57983094222a684cd3f69871ab6084a7a1447c5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 11:39:29 +0000 Subject: [PATCH 1338/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46e0915c8..0a37c9613 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46e0915c89142fb4c8a9f79e23d63cba1b76862f +Subproject commit 0a37c96138270c50f69ae7abdb54a8288b8e9e36 From 1215ca2c4f6924430b9e8608d858098215591748 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:33:16 +0000 Subject: [PATCH 1339/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0a37c9613..4ab612bd4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0a37c96138270c50f69ae7abdb54a8288b8e9e36 +Subproject commit 4ab612bd40b71e17441026c88ab23e3448692e81 From 01f7ff613d498b1346dd0461c1a76c7072810f50 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1340/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4ab612bd4..1db52c531 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4ab612bd40b71e17441026c88ab23e3448692e81 +Subproject commit 1db52c531988dd950c7b68a9ff5c1056b0f97b42 From d94853421163ee9f1879146220ff9a49f591dd16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1341/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ce9da2c5e..206facb47 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ce9da2c5ed3f1b8f4c275c3349a83055a46d3c41 +Subproject commit 206facb472197bc91ceadb837979b187176e40b0 From 944df777149d8465d7dd0266dbbf5e5593504367 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1342/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9021150df..ef18ba7c6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9021150df83050e91ff5c72894de46305d0d54ce +Subproject commit ef18ba7c6f7fb4c045fbe269dc5ab527155fecf0 From 1f1215b91250874c0cefaa25ca4bfaa482d8ef53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1343/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1db52c531..fd5702ff2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1db52c531988dd950c7b68a9ff5c1056b0f97b42 +Subproject commit fd5702ff25dc9b9ea35701615186f41e2606a0d0 From 3486ee77987e41e6b9177d7ca07a54a5e08eb779 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1344/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 206facb47..6747c6826 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 206facb472197bc91ceadb837979b187176e40b0 +Subproject commit 6747c6826ec494f97a1da066d1ad8ed9213fbed7 From 8e3b202a87aace9f6aada13ba84eaca27a79c163 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1345/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index ef18ba7c6..bb698209a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit ef18ba7c6f7fb4c045fbe269dc5ab527155fecf0 +Subproject commit bb698209ad1db770b9284ecd1d2e5eb948e4ce07 From f30bdafb0d69a424a21605e918079420bc477b6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:12:58 +0000 Subject: [PATCH 1346/2295] updated .gitmodules --- .gitmodules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitmodules b/.gitmodules index f5871372d..2bb07ff73 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,6 +9,8 @@ [submodule "sql"] path = sql url = https://github.com/HariSekhon/SQL-scripts + branch = master [submodule "templates"] path = templates url = https://github.com/HariSekhon/Templates + branch = master From dcdc5062136eb9b866425ca33a23967bc8025f34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:13:56 +0000 Subject: [PATCH 1347/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fd5702ff2..62fb7d954 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fd5702ff25dc9b9ea35701615186f41e2606a0d0 +Subproject commit 62fb7d95497cdfcdbfb935240a746e43bbf20502 From fd98dd1225f360a5b6eef455e87a135ec2d7a1c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:14:10 +0000 Subject: [PATCH 1348/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index bb698209a..0ef3948cd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit bb698209ad1db770b9284ecd1d2e5eb948e4ce07 +Subproject commit 0ef3948cd44e56ea996e1de66e708abde7bd4174 From 0d3c5e219cadc09bc29cae2914b10f47ee97d3a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:18:51 +0000 Subject: [PATCH 1349/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 62fb7d954..d65d2027b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 62fb7d95497cdfcdbfb935240a746e43bbf20502 +Subproject commit d65d2027b883addebb8b839c2c7ce8b02f4b31dd From 6db5d0e8373e928fb4e7f272ab8928da98498824 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:42:20 +0000 Subject: [PATCH 1350/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d65d2027b..66bb42884 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d65d2027b883addebb8b839c2c7ce8b02f4b31dd +Subproject commit 66bb4288488e53bda65e249301a88f45fc86242c From 86d55f6d709a00cdb3aea9352aa2144841406fb0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:59:56 +0000 Subject: [PATCH 1351/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 66bb42884..c8671005d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 66bb4288488e53bda65e249301a88f45fc86242c +Subproject commit c8671005d60a0e9574836b60526fbcaefef11c9e From 4604a12d84c268aae044424df96d7dbe8f88c44f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 15:10:04 +0000 Subject: [PATCH 1352/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c8671005d..baaf591a1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c8671005d60a0e9574836b60526fbcaefef11c9e +Subproject commit baaf591a1be91850bce33172539215a7ded76e64 From f8217ecb198d79841a2e66fe44f5b8ec904523e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 16:32:53 +0000 Subject: [PATCH 1353/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index baaf591a1..fa6c84931 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit baaf591a1be91850bce33172539215a7ded76e64 +Subproject commit fa6c84931699372f2f6ec198ed5c13bd3cae4f64 From 90114cc75f11985b8d39db79f0090dc0c18ee714 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 16:56:18 +0000 Subject: [PATCH 1354/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa6c84931..192b08c74 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa6c84931699372f2f6ec198ed5c13bd3cae4f64 +Subproject commit 192b08c74d368b751f9f461a309a45dd884e57ec From 9af717fdef069798ec0bec42529b43562f1370b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 17:00:18 +0000 Subject: [PATCH 1355/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0d4b1863..22edf4ff5 100644 --- a/README.md +++ b/README.md @@ -463,7 +463,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From acbdc9888acc312db3e6bb2c4f2e656587d4e818 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:36:52 +0000 Subject: [PATCH 1356/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 192b08c74..0ad91e913 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 192b08c74d368b751f9f461a309a45dd884e57ec +Subproject commit 0ad91e913edec71d613fb0187d1e24a029e1286f From aa937d84488a7794e26b35967fa917fc77861213 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1357/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0ad91e913..ed73806a8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0ad91e913edec71d613fb0187d1e24a029e1286f +Subproject commit ed73806a8cccc8f09aaa2d15c1f1654b6d197c43 From 634189cfba4eccbe79b75c138c621d7a7c64c341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1358/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6747c6826..d8d872aec 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6747c6826ec494f97a1da066d1ad8ed9213fbed7 +Subproject commit d8d872aec71828b9ecf89e695f10f350ca576b41 From db58b88db0994a18eca633a44e9aded7a1c3158e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1359/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 53c1d53b0..ba7b03f51 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 53c1d53b0ff15f8d9d0b7191b522a1b0b1d0bf20 +Subproject commit ba7b03f51e9a5deb3f9c388e6cbaf1aa2ba6ce3a From fe0a417e7e6c83d82d8f391e727bbe1a06ea3018 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1360/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0ef3948cd..a6808f63e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0ef3948cd44e56ea996e1de66e708abde7bd4174 +Subproject commit a6808f63eea44b4956e49c66e940691ad2fe519b From b571b9c00604c9964b144c0a374b4691d0fa5fb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 11:46:31 +0000 Subject: [PATCH 1361/2295] updated .appveyor.yml --- .appveyor.yml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index eb43a0250..f23536e32 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -41,30 +41,30 @@ on_finish: - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: -# workaround for: -# Some packages could not be installed. This may mean that you have -# requested an impossible situation or if you are using the unstable -# distribution that some required packages have not yet been created -# or been moved out of Incoming. -# The following information may help to resolve the situation: -# -# The following packages have unmet dependencies: -# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed -# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. -# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed -# make[2]: *** [apt-packages] Error 123 -# make[2]: Leaving directory '/home/appveyor/projects/pylib' -# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed -# -# adding "|| :" to the end of these commands causes them to be silently ignored! -- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list -- sudo apt purge -qy --allow-change-held-packages mssql-server -# this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 -#- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- setup/ci_bootstrap.sh -- make + # workaround for: + # Some packages could not be installed. This may mean that you have + # requested an impossible situation or if you are using the unstable + # distribution that some required packages have not yet been created + # or been moved out of Incoming. + # The following information may help to resolve the situation: + # + # The following packages have unmet dependencies: + # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed + # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. + # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed + # make[2]: *** [apt-packages] Error 123 + # make[2]: Leaving directory '/home/appveyor/projects/pylib' + # devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed + # + # adding "|| :" to the end of these commands causes them to be silently ignored! + - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list + - sudo apt purge -yq --allow-change-held-packages mssql-server + # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 + #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages + - setup/ci_bootstrap.sh + - make test_script: -- make test + - make test build: off From d271f6500b45a83c990f67d2c30fea4e7e31ccca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 11:46:50 +0000 Subject: [PATCH 1362/2295] updated .concourse.yml --- .concourse.yml | 76 +++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.concourse.yml b/.concourse.yml index 2abb4ac17..9ed7da3e2 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -14,45 +14,45 @@ # resources: -- name: github - icon: github-circle - type: git - source: - uri: https://github.com/harisekhon/devops-python-tools - branch: master -#- name: daily -# type: time -# source: -# interval: 1d + - name: github + icon: github-circle + type: git + source: + uri: https://github.com/harisekhon/devops-python-tools + branch: master + #- name: daily + # type: time + # source: + # interval: 1d # https://concourse-ci.org/golang-library-example.html jobs: -- name: build - public: false - plan: - - get: github # from resource above - trigger: true - #version: every # build every git commit, default: latest - - task: build - config: - platform: linux - image_resource: - type: docker-image - source: - repository: ubuntu - tag: latest - inputs: - - name: github - path: code - params: - CONCOURSE: 1 - run: - path: /bin/bash - args: - - -c - - | - cd code && - setup/ci_bootstrap.sh && - make init && - make ci test + - name: build + public: false + plan: + - get: github # from resource above + trigger: true + #version: every # build every git commit, default: latest + - task: build + config: + platform: linux + image_resource: + type: docker-image + source: + repository: ubuntu + tag: latest + inputs: + - name: github + path: code + params: + CONCOURSE: 1 + run: + path: /bin/bash + args: + - -c + - | + cd code && + setup/ci_bootstrap.sh && + make init && + make ci test From 1318871cea2024612415948592cd5b98c526fe3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:11:14 +0000 Subject: [PATCH 1363/2295] updated .drone.yml --- .drone.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.drone.yml b/.drone.yml index beda97729..790bdcf4c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,5 @@ +--- +# XXX: putting this separator further down with code causes a parsing bug in drone lint # # Author: Hari Sekhon # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) @@ -28,16 +30,16 @@ type: docker name: default steps: -- name: build - image: ubuntu:18.04 -# environment: -# DEBUG: 1 - commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test + - name: build + image: ubuntu:18.04 + #environment: + # DEBUG: 1 + commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test trigger: branch: - - master + - master From fb3eab1a9369d6119098631c0c80d5007f119375 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:11:49 +0000 Subject: [PATCH 1364/2295] updated .gocd.yml --- .gocd.yml | 115 +++++++++++++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/.gocd.yml b/.gocd.yml index 54f7a3067..69d289bf4 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -16,6 +16,7 @@ # https://docs.gocd.org/current/configuration/configuration_reference.html +--- format_version: 3 pipelines: devops-python-tools: @@ -30,60 +31,60 @@ pipelines: auto_update: true branch: master stages: - - build-and-test: - fetch_materials: true - keep_artifacts: false - clean_workspace: false - approval: - type: success - allow_only_on_success: false - jobs: -# apt-update: -# timeout: 10 -# tasks: -# - exec: -# command: apt -# arguments: -# - update -# run_if: passed -# install-make: -# timeout: 10 -# tasks: -# - exec: -# command: apt -# arguments: -# - install -# - -qy -# - git -# - make -# run_if: passed - ci-bootstrap: - timeout: 10 - tasks: - - exec: - command: setup/ci_bootstrap.sh - run_if: passed - init: - timeout: 10 - tasks: - - exec: - command: make - arguments: - - init - run_if: passed - build: - timeout: 60 - tasks: - - exec: - command: make - arguments: - - ci - run_if: passed - test: - timeout: 60 - tasks: - - exec: - command: make - arguments: - - test - run_if: passed + - build-and-test: + fetch_materials: true + keep_artifacts: false + clean_workspace: false + approval: + type: success + allow_only_on_success: false + jobs: + #apt-update: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - update + # run_if: passed + #install-make: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - install + # - -qy + # - git + # - make + # run_if: passed + ci-bootstrap: + timeout: 10 + tasks: + - exec: + command: setup/ci_bootstrap.sh + run_if: passed + init: + timeout: 10 + tasks: + - exec: + command: make + arguments: + - init + run_if: passed + build: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - ci + run_if: passed + test: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - test + run_if: passed From 73982317e98531a54bd6d1ad86163b680c4e4e44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:12:11 +0000 Subject: [PATCH 1365/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d8f0e09ac..50f6cccb2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ # https://aka.ms/yaml trigger: -- master + - master pool: # there is no /dev/stderr on this azure build! @@ -28,22 +28,22 @@ pool: #container: ubuntu:18.04 steps: -# requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' -- script: env | sort - displayName: env + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' + - script: env | sort + displayName: env -# doesn't work in container due to unprivileged execution and lack of sudo -#- script: sudo apt-get update && sudo apt-get install -y git make -# displayName: install git & make + # doesn't work in container due to unprivileged execution and lack of sudo + #- script: sudo apt-get update && sudo apt-get install -y git make + # displayName: install git & make -#- script: make -# displayName: build + #- script: make + # displayName: build -# doesn't work in vmImage build due to lack of access to normal /dev/stderr device -# tee: /dev/stderr: No such device or address -#- script: make test -# displayName: test + # doesn't work in vmImage build due to lack of access to normal /dev/stderr device + # tee: /dev/stderr: No such device or address + #- script: make test + # displayName: test -# hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( -- script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' - displayName: docker build + # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( + - script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' + displayName: docker build From 81981b20d2debe8f9179e33c463d88b5583fef21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:14:17 +0000 Subject: [PATCH 1366/2295] updated config.yml --- .circleci/config.yml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69f7a6179..80c4cfbb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,27 +14,32 @@ # https://www.linkedin.com/in/harisekhon # -# https://circleci.com/docs/2.0/configuration-reference +# Master Template with more advanced config: +# +# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml + +# Reference: +# +# https://circleci.com/docs/2.0/configuration-reference version: 2.1 + +workflows: + version: 2 + workflow: + jobs: + - build + jobs: build: - # technically a docker image is a better choice than machine - # but we want to introduce some native environment variation - # between build systems in order to test the repo's build automation is robust - machine: - image: default - #image: ubuntu-1604:201903-01 - # set to an actual docker image when running locally using circle_ci_job.sh - # docker image must have git installed to do the checkout - # so using harisekhon/dev:ubuntu instead of base ubuntu image - #image: harisekhon/dev:ubuntu + docker: + - image: cimg/base:2021.04 + resource_class: medium steps: - checkout + - setup_remote_docker: + version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make - # to allow docker networking to work - - run: sudo sysctl net.ipv4.ip_forward=1 - - run: sudo service docker restart - run: make test From e6aa90e89dd74daf8a237c04479e514854421f44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:15:49 +0000 Subject: [PATCH 1367/2295] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80c4cfbb5..37332c73c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: build: docker: - image: cimg/base:2021.04 - resource_class: medium + resource_class: small steps: - checkout - setup_remote_docker: From 8c9ee8f44f659113d1e74616378f355f46713b33 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:23:03 +0000 Subject: [PATCH 1368/2295] updated .travis.yml --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0b4f9044..71348b0bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ # https://docs.travis-ci.com/user/customizing-the-build/ +--- version: ~> 1.0 # ============================ @@ -126,10 +127,10 @@ before_cache: cache: - pip - directories: - - $HOME/.cache - - $HOME/.cpan - - $HOME/.cpanm - - $HOME/.gem + - $HOME/.cache + - $HOME/.cpan + - $HOME/.cpanm + - $HOME/.gem # ============================================== # https://docs.travis-ci.com/user/job-lifecycle/ From ea9a351a3debdf52b6c9da7bf2914f47c9cf81c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 13:55:24 +0000 Subject: [PATCH 1369/2295] added validate.yaml --- .github/workflows/validate.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/validate.yaml diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 000000000..e7e6e2306 --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,33 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: Validation + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * 1' + +jobs: + validate: + name: Validate + uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From 3d27f42fb9d6cd593d3cbcb9e43b9d62054cc5c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:42 +0000 Subject: [PATCH 1370/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ed73806a8..429a989d4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ed73806a8cccc8f09aaa2d15c1f1654b6d197c43 +Subproject commit 429a989d41baa9926c00a7172cfa220927139575 From 5f3daceaa563fe7a18109f02ca0bde833f90575f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:42 +0000 Subject: [PATCH 1371/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d8d872aec..6b5c9b93e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d8d872aec71828b9ecf89e695f10f350ca576b41 +Subproject commit 6b5c9b93e107a321cbd1000aecbff96bf7fe952c From a9797205d5af51ed3819789d26537a83da10d8af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:43 +0000 Subject: [PATCH 1372/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a6808f63e..4893b0b33 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a6808f63eea44b4956e49c66e940691ad2fe519b +Subproject commit 4893b0b33b0d95ddc5549e8b58dcfd0d746d9d15 From 82f29b81f9183b30120b49953c501af24d5d6295 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:29:48 +0000 Subject: [PATCH 1373/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 429a989d4..a2c295fb4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 429a989d41baa9926c00a7172cfa220927139575 +Subproject commit a2c295fb4e04be4af767a69b70bcceeab164a2f7 From 66dafe2315dfb4d1d14f55df7590539ee909c401 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:35:56 +0000 Subject: [PATCH 1374/2295] updated compile.sh --- tests/compile.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/compile.sh b/tests/compile.sh index e235cfd70..952836a20 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; -. ./tests/utils.sh +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" hr echo "Compiling all Python files" hr echo -for x in $(find . -iname '*.py' -o -iname '*.jy'); do - isExcluded "$x" && continue - echo "compiling $x" - python -m py_compile $x -done +while read -r filename; do + isExcluded "$filename" && continue + echo "compiling $filename" + python -m py_compile "$filename" +done < <(find . -iname '*.py' -o -iname '*.jy') echo echo From 8e32bbd7f49e7f9593c73202574ac4cfe817a2cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:37:52 +0000 Subject: [PATCH 1375/2295] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 8d01e3750..abfd23a73 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -15,19 +15,19 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => JSON" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" From 2d3e6e29332627f1b17b99cb30831d1db917c270 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:38:43 +0000 Subject: [PATCH 1376/2295] updated test_xml_to_yaml.sh --- tests/test_xml_to_yaml.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh index 6f93a6ddf..81167836d 100755 --- a/tests/test_xml_to_yaml.sh +++ b/tests/test_xml_to_yaml.sh @@ -15,19 +15,19 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => YAML" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" From 7d5a3d5cb3677f8a92db35be6984667e725ba0a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:40:14 +0000 Subject: [PATCH 1377/2295] updated all.sh --- tests/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index b1e1957dc..103094127 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -29,8 +29,9 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; +# shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. bash-tools/check_all.sh +. "$srcdir/../bash-tools/check_all.sh" #tests/test_yamllint.sh From 89c8f5ea5b713bb9f2a956f69a496a87c081d6c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:44:42 +0000 Subject: [PATCH 1378/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22edf4ff5..678fdf317 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) -[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,6 +54,8 @@ Hari Sekhon - DevOps Python Tools [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) +[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) +[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From d4716aa6a1d9e9e7c3dab91e7e530d0940c4a46b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 15:03:50 +0000 Subject: [PATCH 1379/2295] updated .ipython-notebook-pyspark.00-pyspark-setup.py --- .ipython-notebook-pyspark.00-pyspark-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py index 4dde897da..1de53b3fb 100644 --- a/.ipython-notebook-pyspark.00-pyspark-setup.py +++ b/.ipython-notebook-pyspark.00-pyspark-setup.py @@ -24,4 +24,4 @@ sys.path.insert(0, os.path.join(spark_home, 'python')) for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')): sys.path.insert(0, lib) -execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) +execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable From f3e1e2727ef4372f978dd6db4e34bb2bbfcc9226 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 15:35:52 +0000 Subject: [PATCH 1380/2295] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfef9fec9..c14c0ec49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ avro==1.8.1 # requires Python == 3.4, build in Makefile instead #avro-python3==1.9.0 -awscli==1.16.241 +# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/install_aws_cli.sh +#awscli==1.16.241 #bitarray==0.8.1 #boto==2.49.0 boto3==1.10.37 From 50696dcbea0700c9fc6bf53f47c8d4c335a53826 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 18:37:21 +0000 Subject: [PATCH 1381/2295] updated validate.yaml --- .github/workflows/validate.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index e7e6e2306..81196da71 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -24,7 +24,6 @@ on: - main workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - cron: '0 0 * * 1' jobs: From d62ecad232086b6c2785eabb0141122711c15076 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 19:28:27 +0000 Subject: [PATCH 1382/2295] added semgrep.yaml --- .github/workflows/semgrep.yaml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/semgrep.yaml diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml new file mode 100644 index 000000000..3b145d0d1 --- /dev/null +++ b/.github/workflows/semgrep.yaml @@ -0,0 +1,43 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# S e m g r e p G i t H u b W o r k f l o w +# ============================================================================ # + +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions + +--- +name: Semgrep + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + semgrep: + name: Semgrep + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permissions: + actions: read + contents: read + security-events: write From 5a52ecf8835f81053d4fa1c9e9e0d7dc3f585d8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1383/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a2c295fb4..41ad03ead 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a2c295fb4e04be4af767a69b70bcceeab164a2f7 +Subproject commit 41ad03eadbb7c87a1d65f171ff1c09a7e905246a From 6c25c1581d9f84b05e4f2748d441aef9fbec9702 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1384/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6b5c9b93e..40556d510 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6b5c9b93e107a321cbd1000aecbff96bf7fe952c +Subproject commit 40556d510e99fb25ca190376d278726c4a743a8e From 7ab1f1b803880bcd0d257c217d3ce5a936b3f5b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1385/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ba7b03f51..c19c3de26 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ba7b03f51e9a5deb3f9c388e6cbaf1aa2ba6ce3a +Subproject commit c19c3de26a87c2885489cd3b383c526ffc2945d9 From 0f98399148d46c44bfe7310a9d981da54781590f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1386/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 4893b0b33..9c48fbbb1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 4893b0b33b0d95ddc5549e8b58dcfd0d746d9d15 +Subproject commit 9c48fbbb1235a0b43ddbb657673a7d0445e81743 From 5624548a68635bb1b735680a47bea1ce7d826151 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:51:20 +0000 Subject: [PATCH 1387/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 41ad03ead..a3172a206 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 41ad03eadbb7c87a1d65f171ff1c09a7e905246a +Subproject commit a3172a206422fa1a2626ef5c422d41729c11dcb7 From 177671824ce3f9fe1cdc56f6e8ee896d2c5f86c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:53:18 +0000 Subject: [PATCH 1388/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3172a206..befa8f0bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3172a206422fa1a2626ef5c422d41729c11dcb7 +Subproject commit befa8f0bdf7f366e41bda38379fc7865d4769cda From 244a496e95dd75086b9f0752747d410001c6eb5e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:53:02 +0000 Subject: [PATCH 1389/2295] added semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/semgrep-cloud.yaml diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml new file mode 100644 index 000000000..df624fb33 --- /dev/null +++ b/.github/workflows/semgrep-cloud.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# S e m g r e p C l o u d W o r k f l o w +# ============================================================================ # + +# Logs results to https://semgrep.dev/ + +--- +name: Semgrep Cloud + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + semgrep: + name: Semgrep Cloud + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master From 7be35071169ff96605baca0ddec79e3ce4de948e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:53:40 +0000 Subject: [PATCH 1390/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 3b145d0d1..b319f2356 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -15,6 +15,8 @@ # S e m g r e p G i t H u b W o r k f l o w # ============================================================================ # +# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts + # https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions --- @@ -37,7 +39,3 @@ jobs: semgrep: name: Semgrep uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permissions: - actions: read - contents: read - security-events: write From ec70d959f23e080eadec7ef349347d5938006ecb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:56:39 +0000 Subject: [PATCH 1391/2295] added checkov.yaml --- .github/workflows/checkov.yaml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/checkov.yaml diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml new file mode 100644 index 000000000..90ae12ff5 --- /dev/null +++ b/.github/workflows/checkov.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# C h e c k o v G i t H u b W o r k f l o w +# ============================================================================ # + +# Static analysis of Terraform code - publishes report to GitHub Security tab + +# https://github.com/bridgecrewio/checkov-action + +--- +name: Checkov + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + checkov: + name: Checkov + uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master From 28e285f6b11e787dbcbc2b99ab0332e49bd93a96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:07:57 +0000 Subject: [PATCH 1392/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 678fdf317..2445c90db 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,6 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) -[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) @@ -68,6 +67,8 @@ Hari Sekhon - DevOps Python Tools [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) +[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) From 1fb2a04b416bfba1e7e8da08b9dcd97b771c5ae8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1393/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index befa8f0bd..14c0dc05b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit befa8f0bdf7f366e41bda38379fc7865d4769cda +Subproject commit 14c0dc05b6fa1f41b2ccdc44eca76808ff34d893 From c6c5f8de71a28075d6bedd8e837e1ee952546ae9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1394/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 40556d510..e6088b82c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 40556d510e99fb25ca190376d278726c4a743a8e +Subproject commit e6088b82c68e3edbb89270a388b5d840a66d2838 From 7ef841d48d9aa87cdc6b7b217fdc9f720033cde0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1395/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index c19c3de26..0c8e17fc8 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit c19c3de26a87c2885489cd3b383c526ffc2945d9 +Subproject commit 0c8e17fc883761f43289b7fd834b842ef74f1d8b From 75cc86218273f99c23902ae2e832a179bd9c6445 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1396/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9c48fbbb1..eab20c1ec 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9c48fbbb1235a0b43ddbb657673a7d0445e81743 +Subproject commit eab20c1ec387a76265c2b1e9997a79e6764ab555 From 200fe62138230ada4ecc099b1e81172592153bd4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:19:53 +0000 Subject: [PATCH 1397/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 6550283db..b38994a81 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Alpine -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: alpine - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apk add --no-cache git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Alpine + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + #with: + # debug: 1 From 41b01f5d975ccb0a551828388a5009c0dbb91831 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:20:09 +0000 Subject: [PATCH 1398/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 6c8eada80..dab96c4f0 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Alpine 3 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: alpine:3 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apk add --no-cache git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Alpine 3 + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine3.yaml@master + #with: + # debug: 1 From dda0500f89eb221b555d6b205e4c014411fc5b29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:21:25 +0000 Subject: [PATCH 1399/2295] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 5c214e8fd..494327e4b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest container: centos env: - repo: devops-python-tools + repo: DevOps-Python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From b76491185a7fc4d978bad71540669d2f7981a799 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:22:47 +0000 Subject: [PATCH 1400/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index bb3c76ec6..93b146910 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest container: centos:7 env: - repo: devops-python-tools + repo: DevOps-Python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From 300f7bb3d954a8677ef4dbf05feb351b21ff8287 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:58 +0000 Subject: [PATCH 1401/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14c0dc05b..388c71d13 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14c0dc05b6fa1f41b2ccdc44eca76808ff34d893 +Subproject commit 388c71d13b511ce05230dd12b1602e1775e245c3 From 2ea8ebb0db695a09064d5229061e6e7b7e121ad4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:58 +0000 Subject: [PATCH 1402/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e6088b82c..bc4f4309c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e6088b82c68e3edbb89270a388b5d840a66d2838 +Subproject commit bc4f4309ca17c0a0d9de4f656b2fbde6bb261a3a From 27dbaebc6036ae0ffa1b3a23e6ebbd9ba1887814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:59 +0000 Subject: [PATCH 1403/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index eab20c1ec..71efdf44a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit eab20c1ec387a76265c2b1e9997a79e6764ab555 +Subproject commit 71efdf44afd970b14c24013dc669d876eaf86417 From 3b051d54d578465a90b73b8499c2d660732036fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:50 +0000 Subject: [PATCH 1404/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 388c71d13..f807763cc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 388c71d13b511ce05230dd12b1602e1775e245c3 +Subproject commit f807763cc7b6380588de492d0fe3f828a9e91b65 From 54dc246a7284dff767396edd81fb7b3ad7496ad1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:50 +0000 Subject: [PATCH 1405/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bc4f4309c..a57363bf3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bc4f4309ca17c0a0d9de4f656b2fbde6bb261a3a +Subproject commit a57363bf30e07af15d34364bcdafc960de4cf883 From a9f6519f54d29c2a23396e178ff7368444ce7bfc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:51 +0000 Subject: [PATCH 1406/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 71efdf44a..8f35ff7ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 71efdf44afd970b14c24013dc669d876eaf86417 +Subproject commit 8f35ff7ea92527852dd4cdfe640958b2646ae001 From 6261545c7eefabeae1681e3a6faf1fea534d4dcf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:09:49 +0000 Subject: [PATCH 1407/2295] updated codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 83 ++++++++++++--------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d6aee201a..5e74ab863 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,22 +1,14 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# +--- name: "CodeQL" on: push: - branches: [ master ] + branches: + - master pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: + - master schedule: - cron: '37 15 * * 4' @@ -32,39 +24,40 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - python # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # Command-line programs to run using the OS shell. + # https://git.io/JvXDl + + # If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 95157166937531203f7b5e4e36c4787d9c49bf67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:55 +0000 Subject: [PATCH 1408/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f807763cc..19c99c629 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f807763cc7b6380588de492d0fe3f828a9e91b65 +Subproject commit 19c99c6298a5b2f4d6d099128cc9525c917f0a82 From ebe7a4b691a03134cb6f01271058fc33af0a01bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:55 +0000 Subject: [PATCH 1409/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a57363bf3..23bfcebb9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a57363bf30e07af15d34364bcdafc960de4cf883 +Subproject commit 23bfcebb9d80f042beadc738c44154edd102752f From 5b3cda764fd4d8151078db27ce1881e8317b56be Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:56 +0000 Subject: [PATCH 1410/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8f35ff7ea..1a693b8a1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8f35ff7ea92527852dd4cdfe640958b2646ae001 +Subproject commit 1a693b8a192bec6c1fdf13e55bfe4bbf3d5d1ee4 From a7a8c68ac62ded941d271d5e33a322c7423bebfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 1 Feb 2022 11:55:33 +0000 Subject: [PATCH 1411/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index b319f2356..fcf2abe13 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,5 +37,9 @@ on: jobs: semgrep: - name: Semgrep + name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permission: + actions: read + contents: read + security-events: write From 2a402bfdf520c1c18139bcb0d4940ec18227ff42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:00:53 +0000 Subject: [PATCH 1412/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index b319f2356..fcf2abe13 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,5 +37,9 @@ on: jobs: semgrep: - name: Semgrep + name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permission: + actions: read + contents: read + security-events: write From d97cc000ee941182b6185d0cf3ebde26eedf0b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:01:17 +0000 Subject: [PATCH 1413/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index fcf2abe13..23a921d88 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -39,7 +39,7 @@ jobs: semgrep: name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permission: + permissions: actions: read contents: read security-events: write From dba7169a083dcedb299f35b6cc8090ba60734c1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1414/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 19c99c629..81d8e5d52 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 19c99c6298a5b2f4d6d099128cc9525c917f0a82 +Subproject commit 81d8e5d524995f05c4500f0a395edaba780b6f7e From ccb267a1bebe8ec83f02f55b7dce26f720b8aae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1415/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 23bfcebb9..baffc23ac 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 23bfcebb9d80f042beadc738c44154edd102752f +Subproject commit baffc23aca45c259debcafc1e0bc80d0b22d8345 From 5c09b1af0661af032bb68b73bf0585e31bb9d4dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1416/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0c8e17fc8..5c79c102c 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0c8e17fc883761f43289b7fd834b842ef74f1d8b +Subproject commit 5c79c102ca4aadfca314f9c098acb55d2b9bfbe7 From c9a29a5520326de018a595765990ed254475261b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1417/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 1a693b8a1..bdebc016d 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 1a693b8a192bec6c1fdf13e55bfe4bbf3d5d1ee4 +Subproject commit bdebc016d72178a97ef09c7593e5157e15d1f9d9 From 4e2c5aee63eb7068420e240b2a0ae685f5beab56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:06:46 +0000 Subject: [PATCH 1418/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index fcf2abe13..23a921d88 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -39,7 +39,7 @@ jobs: semgrep: name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permission: + permissions: actions: read contents: read security-events: write From 445507835f9f8592c9510ea86fd0cf9016dcea0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:08:03 +0000 Subject: [PATCH 1419/2295] updated bootstrap.sh --- setup/bootstrap.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 5383ae1f0..306e8ecb4 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -15,11 +15,11 @@ # Alpine / Wget: # -# wget https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh && sh bootstrap.sh +# wget -O- https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh # # Curl: # -# curl https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh +# curl https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh set -eu [ -n "${DEBUG:-}" ] && set -x @@ -40,16 +40,24 @@ if [ "$(uname -s)" = Darwin ]; then elif [ "$(uname -s)" = Linux ]; then echo "Bootstrapping on Linux: $repo" if type apk >/dev/null 2>&1; then - $sudo apk --no-cache add bash git make curl + $sudo apk --no-cache add bash git make curl wget elif type apt-get >/dev/null 2>&1; then + if [ -n "${CI:-}" ]; then + export DEBIAN_FRONTEND=noninteractive + fi opts="" if [ -z "${PS1:-}" ]; then opts="-qq" fi $sudo apt-get update $opts - $sudo apt-get install $opts -y git make curl + $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then - $sudo yum install -y git make curl + if grep -qi 'NAME=.*CentOS' /etc/*release; then + echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs" + $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-* + $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-* + fi + $sudo yum install -y git make curl wget else echo "Package Manager not found on Linux, cannot bootstrap" exit 1 From 64829ab4b40ded3fd2e7518565e360d7bebf1184 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:06 +0000 Subject: [PATCH 1420/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 81d8e5d52..36728e719 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 81d8e5d524995f05c4500f0a395edaba780b6f7e +Subproject commit 36728e7191402d862c58732a839bda2db604699c From c9960ebe9c6311b42a04ec7b03a9e0261b24df20 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:06 +0000 Subject: [PATCH 1421/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index baffc23ac..cf61b55f6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit baffc23aca45c259debcafc1e0bc80d0b22d8345 +Subproject commit cf61b55f6ae953eabafaff97a71e94b3983d21c1 From 1ad2ea765cef0bcded1fa20a5070036874fc2bc7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:07 +0000 Subject: [PATCH 1422/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 5c79c102c..2926bf822 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 5c79c102ca4aadfca314f9c098acb55d2b9bfbe7 +Subproject commit 2926bf822469f9a2d2bda6fb5705bf3b6ed52358 From ac05856fa6a2bf46e69d48c23f4d86f690093ea5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:07 +0000 Subject: [PATCH 1423/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index bdebc016d..054e52071 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit bdebc016d72178a97ef09c7593e5157e15d1f9d9 +Subproject commit 054e52071c6e9b48e8603e7abb28ce7ae89d05cb From 5f8ea90955bc40e6ccd084522312bdb673509269 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 12:08:51 +0000 Subject: [PATCH 1424/2295] updated LICENSE --- LICENSE | 46 ++++------------------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/LICENSE b/LICENSE index 03b527051..4860c1cf5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,45 +1,7 @@ -======================================= -HARI SEKHON LICENSE Revision 2013112300 -======================================= +Copyright 2015 Hari Sekhon -Copyright (c) 2006 onwards, Hari Sekhon -All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -Redistribution and use in source and binary forms, with or without -modification, is permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. All advertising materials mentioning features or use of this software - must display the following acknowledgement: - This product includes software developed by Hari Sekhon. -4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or - promote products derived from this software without specific prior written - permission. -5. Modifications may be released to the public only with prior written permission - from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating - patch pull requests back to the original repository. -6. Private modifications may be made to suit requirements, but any modifications - to this work, whether publicly disclosed or not, must be sent back to - Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or - LinkedIn (https://www.linkedin.com/in/harisekhon) - and must come under this same license. Any such modifications may be - reincorporated for the improvement of this software. -7. This work may not be sold without prior written permission from Hari Sekhon -8. This license may change at any time and the latest revision supersedes - all prior revisions. -9. Alternative licensing must be agreed in writing with Hari Sekhon prior - to public availability. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 10457e5b14e2d761189ca5bada6cbf55a3ccfbd0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 15:52:32 +0000 Subject: [PATCH 1425/2295] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 90ae12ff5..a31ba133a 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -39,3 +39,7 @@ jobs: checkov: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master + permissions: + actions: read + contents: read + security-events: write From 5c6837cf2344cb3119450e9c5f349a7ab121325a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 16:04:13 +0000 Subject: [PATCH 1426/2295] updated validate.yaml --- .github/workflows/validate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 81196da71..cf901ed71 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,7 @@ # https://www.linkedin.com/in/harisekhon # +--- name: Validation on: From 230ce8fdfd038b0637fb0355fa090db181047d3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 16:10:58 +0000 Subject: [PATCH 1427/2295] removed .github/workflows/centos6.yaml.disabled --- .github/workflows/centos6.yaml.disabled | 49 ------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/centos6.yaml.disabled diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled deleted file mode 100644 index b78c9de3a..000000000 --- a/.github/workflows/centos6.yaml.disabled +++ /dev/null @@ -1,49 +0,0 @@ -# -# Author: Hari Sekhon -# Date: Tue Feb 4 09:53:28 2020 +0000 -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/harisekhon/devops-python-tools -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback -# -# https://www.linkedin.com/in/harisekhon -# - -# Not supporting RHEL6 any more because it doesn't have GNU parallel package - -name: CentOS 6 - -#env: -# DEBUG: 1 - -on: - push: - branches: - - master - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' - -jobs: - build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:6 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test From bd11ceed0dc87cea96b00e484e53c7f4a3faf858 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:17:49 +0000 Subject: [PATCH 1428/2295] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index df624fb33..d18460b45 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -37,3 +37,5 @@ jobs: semgrep: name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master + secrets: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} From 6b2dad16f356bc9adb5ebd3443080313333ed138 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:18:56 +0000 Subject: [PATCH 1429/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 23a921d88..af3635d78 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,7 +37,7 @@ on: jobs: semgrep: - name: Semgrep GitHub + name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: actions: read From 03118a276424c549ee221bba149e30e23354b4b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:24 +0000 Subject: [PATCH 1430/2295] updated centos.yaml --- .github/workflows/centos.yaml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 494327e4b..f4bf8241b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -11,37 +11,20 @@ # https://www.linkedin.com/in/harisekhon # +--- name: CentOS -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos - env: - repo: DevOps-Python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 882930ebfe12d704d45c3d99fc0933e336b29ae4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:24 +0000 Subject: [PATCH 1431/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 93b146910..3f8e5b283 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -11,37 +11,20 @@ # https://www.linkedin.com/in/harisekhon # +--- name: CentOS 7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:7 - env: - repo: DevOps-Python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS 7 + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 3f4a12e3b13617cf6f896ed1f8b4d615a7432798 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:25 +0000 Subject: [PATCH 1432/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 070024988..255c2c66e 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: CentOS 8 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:8 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS 8 + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 69947f9f254dcc665bb3216312886f9f28aa6608 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:25 +0000 Subject: [PATCH 1433/2295] updated debian.yaml --- .github/workflows/debian.yaml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d729ae64e..a150bb2eb 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + #with: + # debug: 1 From 63b5dbcb3b876b2763ea8954beeb0ed1d0a2a502 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1434/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 35 ++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 454fe8bb2..68e161841 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 10 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:10 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 10 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian10.yaml@master + #with: + # debug: 1 From 712cb22f3ddb09b22df69000a420215fa5b6eddf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1435/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 46 ++++++------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 96673f75d..c2d8e7078 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -4,53 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # -name: CI Debian 6 +--- +name: Debian 6 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 10 - runs-on: ubuntu-latest - container: debian:6 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - echo VERSION: ; cat /etc/*release /etc/*version 2>/dev/null || : - apt-get update && - apt-get install -y git make - - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" - - name: build - run: | - cd "/tmp/$repo" && - make - - name: test - run: | - cd "/tmp/$repo" && - make test + name: Debian 6 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian6.yaml@master + #with: + # debug: 1 From d9f55c88c18f604f9cc29aed9a9b886368c8ca75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1436/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 46 ++++++------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 622715687..f6329c136 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -4,53 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # -name: CI Debian 7 +--- +name: Debian 7 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 10 - runs-on: ubuntu-latest - container: debian:7-slim - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make - - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" - - name: build - run: | - cd "/tmp/$repo" && - make - - name: test - run: | - cd "/tmp/$repo" && - make test + name: Debian 7 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian7.yaml@master + #with: + # debug: 1 From 360303edbed9b88627f2dd715f8575dbe7076cc7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:27 +0000 Subject: [PATCH 1437/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 35 +++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 99703fe57..fa99760a7 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 8 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:8 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 8 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian8.yaml@master + #with: + # debug: 1 From 88bf873e28d04fb9c0c04f502137aa46bf0e790d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:27 +0000 Subject: [PATCH 1438/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 35 +++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index db7819b07..e58fd190b 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 9 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:9 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 9 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian9.yaml@master + #with: + # debug: 1 From 9bbe1cf93dcdb39440869d56b274933eb99378b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:28 +0000 Subject: [PATCH 1439/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index c7ca8aaae..0d13ef9b0 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Fedora -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: fedora - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Fedora + uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master + #with: + # debug: 1 From 2707ee95abdac3719f94e919f9178774a2c0d8bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:28 +0000 Subject: [PATCH 1440/2295] updated mac.yaml --- .github/workflows/mac.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 02cd075cd..c2216dc2d 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Mac -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: brew update - run: which brew && brew update || echo - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Mac + uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + #with: + # debug: 1 From 443e322d00fd214023cfa6f9466f26cf8f328705 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1441/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 33 ++++++++------------------------ 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 392688e0a..ec1d7a6f8 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Mac 10.15 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: macos-10.15 - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: brew update - run: which brew && brew update || echo - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Mac 10.15 + uses: HariSekhon/GitHub-Actions/.github/workflows/mac10.15.yaml@master + #with: + # debug: 1 From a6ac95e17dfc62dab7244a273d5b2056c0fadee5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1442/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 38 +++++++----------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index f605de0f2..152523c3b 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -13,41 +13,17 @@ name: PyPy 2 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [pypy2] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: PyPy2 + uses: HariSekhon/GitHub-Actions/.github/workflows/pypy2.yaml@master + #with: + # debug: 1 From 468faaa047d756597b2fede7e4459a7bc829ace8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1443/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 38 +++++++----------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 1cb9ef4f7..280145926 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -13,41 +13,17 @@ name: PyPy 3 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [pypy3] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: PyPy3 + uses: HariSekhon/GitHub-Actions/.github/workflows/pypy3.yaml@master + #with: + # debug: 1 From 03c6cf189b6d1cc230cc0136e9b403877d7b5ca8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1444/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 485e4859b..c57282060 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 2.7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [2.7] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 2.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/python2.7.yaml@master + #with: + # debug: 1 From 09849af56b0e76a06ed4f181065fc293fcd67b43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1445/2295] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 15fa37297..a8b945742 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.5 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.5] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.5 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.5.yaml@master + #with: + # debug: 1 From 70226d805c18f241f2f2b8f3d17c87aa40d2e3e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1446/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 8eeb1a0e3..b9b3ab180 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.6 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.6] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.6 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.6.yaml@master + #with: + # debug: 1 From a74ef1b4d904106501c021ec0100ff6abb1cc3c9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:31 +0000 Subject: [PATCH 1447/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 5bc6252f7..57720b5c3 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.7] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.7.yaml@master + #with: + # debug: 1 From 2eb63a26fc41a2c3466eebd068630494f55a8a64 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:31 +0000 Subject: [PATCH 1448/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 0764a94dc..2b7abf97e 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.8 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.8] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.8 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.8.yaml@master + #with: + # debug: 1 From 86241a549e8fb3ebec8447679d2966c5ad44f4e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:32 +0000 Subject: [PATCH 1449/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index cac47c8a3..204bf7572 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:latest - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + #with: + # debug: 1 From cd0b50a1639e7319a59c5aba0d1bc5f26cb36bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:32 +0000 Subject: [PATCH 1450/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 8e267edb4..3e01a484e 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 14.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:14.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 14.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu14.yaml@master + #with: + # debug: 1 From fbd61be719f394b6bcbf33876cfcbd2253a01d30 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:33 +0000 Subject: [PATCH 1451/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5db70c05..8184d3801 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 16.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:16.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 16.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu16.yaml@master + #with: + # debug: 1 From 00b0d4bb64d5dc40f8091d8d18cc8adcc9f88966 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:54 +0000 Subject: [PATCH 1452/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 2bf3b18a7..7d8cf7e9c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 18.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:18.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 18.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu18.yaml@master + #with: + # debug: 1 From 08853eccd4e7045ac2f8a89d72e03862e05de81a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:33:22 +0000 Subject: [PATCH 1453/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index bc0c83aac..aa7c0e846 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 20.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:20.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 20.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu20.yaml@master + #with: + # debug: 1 From 1eeff6d3844828556254f7fb6c50c9e979c32676 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:33:23 +0000 Subject: [PATCH 1454/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 33 ++++++++-------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 08b8f24a3..103fa566b 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -4,42 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: GitHub Actions Ubuntu -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: GitHub Ubuntu + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master + #with: + # debug: 1 From 3916d589775b46e3ba38044e7b6d4504e4a3b380 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1455/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36728e719..385f229d4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36728e7191402d862c58732a839bda2db604699c +Subproject commit 385f229d40c19eaf7962577244f1967033c3dfbf From 4750f61a3971ab03dfaea4addcbab48ecc0a1be1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1456/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cf61b55f6..22ad03fa9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cf61b55f6ae953eabafaff97a71e94b3983d21c1 +Subproject commit 22ad03fa9aa7627c3a8971c4f3515b067d124121 From 2202b676bee2d9ef9a08702161272c5d2408147b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1457/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 2926bf822..731f3d458 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 2926bf822469f9a2d2bda6fb5705bf3b6ed52358 +Subproject commit 731f3d4583b49881c1a267874acfaa6cfa6eb7e0 From d96d5348a90ea8d0b0bfef6301b38b208e5e74b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1458/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 054e52071..edb86249c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 054e52071c6e9b48e8603e7abb28ce7ae89d05cb +Subproject commit edb86249ca9efcea597a9dfd072097a4536734e5 From ef6e3c679bc3aaf230a2bec6e37c68e2cb246ccd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1459/2295] added dockerhub_pytools_alpine.yaml --- .../workflows/dockerhub_pytools_alpine.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_alpine.yaml diff --git a/.github/workflows/dockerhub_pytools_alpine.yaml b/.github/workflows/dockerhub_pytools_alpine.yaml new file mode 100644 index 000000000..ff2197707 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_alpine.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Alpine) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: alpine + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-alpine + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 4b46521c07c2da96ff0a61a6f37310366f238741 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1460/2295] added dockerhub_pytools_centos.yaml --- .../workflows/dockerhub_pytools_centos.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_centos.yaml diff --git a/.github/workflows/dockerhub_pytools_centos.yaml b/.github/workflows/dockerhub_pytools_centos.yaml new file mode 100644 index 000000000..06778bd22 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_centos.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (CentOS) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: latest centos + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-centos + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From f681cac11224c027728dde5824e2497880a33a08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1461/2295] added dockerhub_pytools_debian.yaml --- .../workflows/dockerhub_pytools_debian.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_debian.yaml diff --git a/.github/workflows/dockerhub_pytools_debian.yaml b/.github/workflows/dockerhub_pytools_debian.yaml new file mode 100644 index 000000000..aa9da7e6a --- /dev/null +++ b/.github/workflows/dockerhub_pytools_debian.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Debian) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: debian + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-debian + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From b851f84d1b22ea1a9ed597b79c7d11ed9b6755d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:11 +0000 Subject: [PATCH 1462/2295] added dockerhub_pytools_fedora.yaml --- .../workflows/dockerhub_pytools_fedora.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_fedora.yaml diff --git a/.github/workflows/dockerhub_pytools_fedora.yaml b/.github/workflows/dockerhub_pytools_fedora.yaml new file mode 100644 index 000000000..3aa3c800f --- /dev/null +++ b/.github/workflows/dockerhub_pytools_fedora.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Fedora) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: fedora + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-fedora + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 11fbd0c11c0d5ce35bab6234b38fa731b7629ebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:11 +0000 Subject: [PATCH 1463/2295] added dockerhub_pytools_ubuntu.yaml --- .../workflows/dockerhub_pytools_ubuntu.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_ubuntu.yaml diff --git a/.github/workflows/dockerhub_pytools_ubuntu.yaml b/.github/workflows/dockerhub_pytools_ubuntu.yaml new file mode 100644 index 000000000..d53930274 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_ubuntu.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Ubuntu) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: ubuntu latest + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From a9026344c8b7e174166b5ec1c53fb1881077525f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:26:31 +0000 Subject: [PATCH 1464/2295] added ghcr_python_ubuntu.yaml --- .github/workflows/ghcr_python_ubuntu.yaml | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ghcr_python_ubuntu.yaml diff --git a/.github/workflows/ghcr_python_ubuntu.yaml b/.github/workflows/ghcr_python_ubuntu.yaml new file mode 100644 index 000000000..fdd5ad61b --- /dev/null +++ b/.github/workflows/ghcr_python_ubuntu.yaml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: GHCR Build (Ubuntu) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master + with: + image: pytools + tags: ubuntu latest + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 25e68b03e644a0b96831062c47e9f7bc17c7d5ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1465/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 385f229d4..018732364 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 385f229d40c19eaf7962577244f1967033c3dfbf +Subproject commit 018732364cb003fc7cb63287b0ca08f4aa401b42 From 0f7f46084fdee8fa55cf3e3014893139d1905c3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1466/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 22ad03fa9..8c97585a3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 22ad03fa9aa7627c3a8971c4f3515b067d124121 +Subproject commit 8c97585a367eedb957a25de2ae37582bd8c6f32e From b85cda9a1994aa2b4392618035936a43db3343c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1467/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 731f3d458..103a1543f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 731f3d4583b49881c1a267874acfaa6cfa6eb7e0 +Subproject commit 103a1543fa73a0adf4f0af710fcf04d099500170 From 1f7788deaeb431263a7778a69e94c09223a81210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1468/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index edb86249c..542fac6c4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit edb86249ca9efcea597a9dfd072097a4536734e5 +Subproject commit 542fac6c4e00adcb8b885cceb07b5dfee5361c05 From f0e52eaf57ab9fc3e88accfeae0df0c6f6ceb91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 11:18:24 +0000 Subject: [PATCH 1469/2295] updated requirements.txt --- requirements.txt | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c14c0ec49..6060cbe23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,30 @@ avro==1.8.1 + # requires Python == 3.4, build in Makefile instead #avro-python3==1.9.0 + # AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/install_aws_cli.sh #awscli==1.16.241 #bitarray==0.8.1 + #boto==2.49.0 boto3==1.10.37 #cassandra-driver==3.6.0 dicttoxml==1.7.4 + # Elasticsearch library must match major version :( # for Elasticsearch 2.x #elasticsearch>=2.0.0,<3.0.0 # for Elasticsearch 1.x #elasticsearch>=1.0.0,<2.0.0 + # fails on requiring newer version of setuptools #Flask==0.10.1 GitPython==2.1.15 + # this GCP API is surprisingly awful, not using #google-api-python-client==1.11.0 + happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 @@ -26,36 +33,49 @@ jinja2==2.11.3 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 + # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 + # requires pg_config to build from source #psycopg2==2.8.4 psycopg2-binary==2.8.4 + python-cson==1.0.9 psutil==5.7.0 + # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #PyHive==0.6.1 + +# doesn't work with non-trivial code #PyInstaller==3.3.1 -python-ldap==3.2.0 + +# gcc compile error on Alpine, don't think this is used either +#python-ldap==3.2.0 + #python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] #python-krbV==1.0.90 # needed by avro + python-snappy==0.5 sasl==0.2.1 sh==1.12.14 selenium==3.141.0 + # pulls in python-KrbV as a dependency which doesn't build on Mac any more # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 #snakebite==2.11.0 + thrift-sasl==0.2.1 thrift==0.9.3 thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 + #pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found From cf4afe6858e23005b9c50a0fa6f34e0803909eda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:06 +0000 Subject: [PATCH 1470/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 152523c3b..61d7b8925 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -24,6 +24,7 @@ on: jobs: build: name: PyPy2 - uses: HariSekhon/GitHub-Actions/.github/workflows/pypy2.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: pypy2 + #debug: 1 From c6d05ecb8e505cfc1226e27e1bb3bfb2d8e3fce9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:07 +0000 Subject: [PATCH 1471/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 280145926..1b8cb30ed 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -24,6 +24,7 @@ on: jobs: build: name: PyPy3 - uses: HariSekhon/GitHub-Actions/.github/workflows/pypy3.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: pypy3 + #debug: 1 From 6169e0ea374df4b5249d7633252e178bf3d18744 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:07 +0000 Subject: [PATCH 1472/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c57282060..8fc49fa55 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 2.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python2.7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 2.7 + #debug: 1 From cd2f98dec3d3cbebb4a358dcd659c563f5b7546a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:08 +0000 Subject: [PATCH 1473/2295] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index a8b945742..f77c1fc28 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.5 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.5.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.5 + #debug: 1 From 8ff9c649f169bd195affd96ec3448898784e7774 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:08 +0000 Subject: [PATCH 1474/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b9b3ab180..a661b9493 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,9 @@ on: jobs: build: name: Python 3.6 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.6.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master #with: # debug: 1 + with: + version: 3.5 + #debug: 1 From 6c2f481ccb08abd79ce7a0a4389c100aaa2b33e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:09 +0000 Subject: [PATCH 1475/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 57720b5c3..c621468f0 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.7 + #debug: 1 From feff67e17eb584476d459542bbbcdf2eccb64b90 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:09 +0000 Subject: [PATCH 1476/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 2b7abf97e..636d06d06 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.8 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.8.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.8 + #debug: 1 From 97fe5e5993d895b946f8c100518f05fbefb59791 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:09 +0000 Subject: [PATCH 1477/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index b38994a81..a0a127330 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -26,5 +26,6 @@ jobs: build: name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 9284c551ca19ef799dad9f46d1beb426a7fd9085 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:10 +0000 Subject: [PATCH 1478/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index dab96c4f0..409fd6f68 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine3.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + with: + version: 3 + #debug: 1 From 71b2976aca14b23ad3347a780fe49a1617a7d773 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:10 +0000 Subject: [PATCH 1479/2295] updated centos.yaml --- .github/workflows/centos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index f4bf8241b..10e7cb54a 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 5b03adef213117c1cf3fd33ffc14b73d72b61a25 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:11 +0000 Subject: [PATCH 1480/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 3f8e5b283..ec6f639b1 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: 7 + #debug: 1 From 4d52f5b3a9ab19cabc47bcdb6a4ba13b95898eae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:11 +0000 Subject: [PATCH 1481/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 255c2c66e..713ea3e67 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: 8 + #debug: 1 From ec3ea8273742473f527380fc46ae733f140a461d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:12 +0000 Subject: [PATCH 1482/2295] updated debian.yaml --- .github/workflows/debian.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index a150bb2eb..78fc934f5 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -26,5 +26,6 @@ jobs: build: name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 1a8d7666cd2beed752fb62216ead8cf83c20c940 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:12 +0000 Subject: [PATCH 1483/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 68e161841..ce950e6e8 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian10.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 10 + #debug: 1 From 661e8a8978367f2ca6fee6d28bc11f529ae5b066 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:13 +0000 Subject: [PATCH 1484/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index c2d8e7078..f087db959 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian6.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 6 + #debug: 1 From e5b1c8b5e408089bf27342c9fd0f8c877dfe6e25 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:13 +0000 Subject: [PATCH 1485/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index f6329c136..30677854f 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 7 + #debug: 1 From 0573d4c6038dd7440c29b855df3a2710162a511d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1486/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index fa99760a7..179419b4f 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian8.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 8 + #debug: 1 From f57cd0a7f5efb34605b24b5dca236d5d1e30591c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1487/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index e58fd190b..38e677a76 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian9.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 9 + #debug: 1 From 620d29e47e879b65964cdeb7f31c305271918c2c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1488/2295] updated mac.yaml --- .github/workflows/mac.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index c2216dc2d..0c9454bd6 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -26,5 +26,6 @@ jobs: build: name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From b362f0c5dc86ad9db86792a97f8e040a96b065ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:15 +0000 Subject: [PATCH 1489/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index ec1d7a6f8..e214adb9a 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Mac 10.15 - uses: HariSekhon/GitHub-Actions/.github/workflows/mac10.15.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + with: + version: 10.15 + #debug: 1 From f88475880eefad06c727d8902e0bf294fd7d0b44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:16 +0000 Subject: [PATCH 1490/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index a661b9493..623f85126 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -29,5 +29,5 @@ jobs: #with: # debug: 1 with: - version: 3.5 + version: 3.6 #debug: 1 From 151252eade057c51b1bfda7c681d86d41f438001 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:16 +0000 Subject: [PATCH 1491/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 204bf7572..d5c6df777 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -26,5 +26,6 @@ jobs: build: name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 3a5a85e06a26db6e2c7b8d8f8b910eb62738f921 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:17 +0000 Subject: [PATCH 1492/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 3e01a484e..cda82b285 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu14.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 14.04 + #debug: 1 From 1523c810c31b295ae00960b8930d12743dbc7210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:17 +0000 Subject: [PATCH 1493/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 8184d3801..96b935a17 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu16.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 16.04 + #debug: 1 From 03fae4d5a2e26732bc0683270be6da632644e965 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:18 +0000 Subject: [PATCH 1494/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 7d8cf7e9c..0c7f11c41 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu18.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 18.04 + #debug: 1 From ecc078d643cc40ccb6ed3c3edd28cb964b71d38b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:18 +0000 Subject: [PATCH 1495/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index aa7c0e846..56701d844 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu20.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 20.04 + #debug: 1 From b1cfd31ed7b749c58b6bd881f7551c7457ac35ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:47 +0000 Subject: [PATCH 1496/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a0a127330..1b72eba8d 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Alpine From e72b4d7a4f3646026fea0fdf1821ccc923af7d5e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1497/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 409fd6f68..f041be5d2 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Alpine 3 From 658cddd6bed7feecf08f12b267f469d38bb943fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1498/2295] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 10e7cb54a..5949b16cf 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS From 478e890aff50ed02b081c01cdac42d8e883048c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1499/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index ec6f639b1..77815b813 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS 7 From 0046681a0849112f5a34fcd0912512b4ffa0c2c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1500/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 713ea3e67..4df3a4cba 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS 8 From 83bf8fa04a09067df1c7999fa5e307ab8f2e4daa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1501/2295] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 78fc934f5..13418a6ab 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian From 7383d246bea930dc9599839cbe17732b937aefe9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1502/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index ce950e6e8..8b5c32f96 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 10 From 6f8d28ec9e54820ec4179a85cb92fa6f9d41789f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1503/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index f087db959..5023bbe64 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 6 From 14dcfb70f54abbed925da137abd3f7376d407b8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1504/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 30677854f..232c2893a 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 7 From c4ab9e1ca04bd5e7d518acce84d8cd21f56a33f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1505/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 179419b4f..65bb383ba 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 8 From d67da6d7a5af06423b34835d1816b02306075943 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1506/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 38e677a76..5308750e8 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 9 From 55f744986525aef20bf3adcb57624bb5b74d4e61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1507/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 61d7b8925..52c08058b 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -21,6 +21,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: PyPy2 From a8cb330177767d6023e06c16949576edf5fa5c39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1508/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 1b8cb30ed..62afbfb46 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -21,6 +21,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: PyPy3 From 645f274ea43cdff7a81a78f81a03a21694eb7339 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1509/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 8fc49fa55..8381f774f 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 2.7 From b8237f82369ddef17eb56d68b4ab565dd8bd3590 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1510/2295] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index f77c1fc28..b735cfd0c 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.5 From eb105a91052e39e63202dcb496a9a8dd785d1b0e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1511/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 623f85126..3cedc3c2c 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.6 From 6d08d020ba9bdf257731c25bc6f07b38306f42f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:53 +0000 Subject: [PATCH 1512/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index c621468f0..1414475c1 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.7 From 72b852a90e505f5bc967904050c049de4e9326af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:53 +0000 Subject: [PATCH 1513/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 636d06d06..14bf646cd 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.8 From 142e19864a57032f98390e55fa5c902017db45d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 1514/2295] updated mac.yaml --- .github/workflows/mac.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 0c9454bd6..7224f71c2 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Mac From 94ac28ca6936a6817fd312ac2735a30e4e50ede2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 1515/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index e214adb9a..8753027af 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Mac 10.15 From 4515aac87bbf060dfa23657c3170b89aaa161b66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 1516/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index d5c6df777..5768ae6e7 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu From 0276f66b7797aaae4e5855e0a09ffc9b8da2bda5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 1517/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index cda82b285..428f23491 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 14.04 From 54745a99f7ccd298fd902bd5181fb548be4cdce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 1518/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 96b935a17..b61a9202d 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 16.04 From b050e50986ae4f738ccd534b9c75ff25c1251941 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 1519/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 0c7f11c41..49c5cf96c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 18.04 From 5cfdea1deb89c07d88041a129d3b87fcad993c13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:56 +0000 Subject: [PATCH 1520/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56701d844..41d7d9d3f 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 20.04 From a101b530eee066e481f277fb9bfa70bbd3306349 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:56 +0000 Subject: [PATCH 1521/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 103fa566b..be5a946d8 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: GitHub Ubuntu From bc7a188b8d2f555fdb093fea22276c9860235f54 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:36:56 +0000 Subject: [PATCH 1522/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0d13ef9b0..99b3e8f79 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Fedora From 7e3c85a3d2e1d4d0dc9f4890b314ae6dada1cbef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:18 +0000 Subject: [PATCH 1523/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 1b72eba8d..a92204784 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 9ad486a7b40fabe657f642ff050dfc23459fd6a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:18 +0000 Subject: [PATCH 1524/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index f041be5d2..4b0cb4170 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 71888452fd4ce66f778ac9816adbd0a7a81c07c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1525/2295] updated centos.yaml --- .github/workflows/centos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 5949b16cf..ff1e33598 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 0e4b8776c6dd48cb5c277aacc7fc34b790aef626 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1526/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 77815b813..588f5de59 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From eff5e66fe0406f2ee4aa17f61471c4a260d0296c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1527/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4df3a4cba..23aeb1c79 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From b36e324ec7982d45d664269c5a1f4c4cef055322 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1528/2295] updated debian.yaml --- .github/workflows/debian.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 13418a6ab..0ac7e1f3e 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 0da43423e5c05179ad938e28b8aaeb79de50bf2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1529/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 8b5c32f96..7763b32f2 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 3623d46a49026f7c073e54f3ec26f46b112e60f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1530/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 5023bbe64..99015df89 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From e4ad04e9336418aa45c3f6bbc853406581d057af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1531/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 232c2893a..3dea635f2 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 13fda2a38079378cc47156c3618ccac1737d979d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1532/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 65bb383ba..8d043b023 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 48f562024119d487ec55d74a76a32e40cb2531ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1533/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 5308750e8..6a9c34723 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 12904c3ff136c9ec52890cf16943282f206b429f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1534/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 99b3e8f79..c893bc2cf 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 1879a606cd8836dcbb98f325a9a87ac39ed64ae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1535/2295] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 7224f71c2..ea6c12828 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From af632352f89908ef440908ca63569ad5525e2d0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1536/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 8753027af..b5c50bc4c 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 73c8a1ec8063f5a53846da89294cc2f7a5f25de4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1537/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 52c08058b..489d1bcc4 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -23,7 +23,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 249928a26f492a9a4f611f0d474bbafbfb61760a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 1538/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 62afbfb46..5c0e12095 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -23,7 +23,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 27d3c1170459d081081675da55cb5707fac900ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1539/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 8381f774f..c58d3aaa9 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From bbcf69a3380403d97d99c74331871d2b4c811a80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1540/2295] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index b735cfd0c..659bbc6f6 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 255b2e110bd07e67a9912507db2d5ab66adfb09f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1541/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 3cedc3c2c..b325c9523 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 504a7fd05586cbd767996eef43c4eb85440a10e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1542/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 1414475c1..413e03916 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 9d58ee7f5a8580e3b87db085929994bc73124ee4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1543/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 14bf646cd..9621587bb 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From af83242bba9e9d4ee30fb1bd20fa186db391054d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1544/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5768ae6e7..07c4af459 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 04f01563b59622c7da5ca4d275f51f83570eb540 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1545/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 428f23491..561d7cfda 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From b17c666e25d841628123c0c58479bbffc9ac5c96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1546/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index b61a9202d..59d45e0ae 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 2fbd28187fcb209637d067315f4e4f3c9126062a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1547/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 49c5cf96c..d3818d27d 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 996ea9a0b800e4fc974d3fc25f8385e3e5b57ed0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1548/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 41d7d9d3f..7ceee32d7 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 79c9b6a7f4840eb15f24ce8e07768eaea360e314 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 1549/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index be5a946d8..4c463496a 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 3b37de4bff5f37abf98f65b9d51536b568e4fe74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 1550/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 018732364..e71685633 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 018732364cb003fc7cb63287b0ca08f4aa401b42 +Subproject commit e71685633a9bf9d8aa19dc9f9597194e4ce5eae4 From 2269bb68cf2f48e114f1ef3a9654de697f87ab6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 1551/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8c97585a3..4fa30554f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8c97585a367eedb957a25de2ae37582bd8c6f32e +Subproject commit 4fa30554fc746d5eb90e374c7782bc1bef7f6c47 From 493f0d25b80d28fdbeb32c00f7595bf0bd7a6e83 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 1552/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 542fac6c4..9b14e6b9b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 542fac6c4e00adcb8b885cceb07b5dfee5361c05 +Subproject commit 9b14e6b9be41fd29ba76a585bd490721be05b2d6 From 1520679f301aa861d16c8b709d63ed4a90413668 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Feb 2022 17:29:09 +0000 Subject: [PATCH 1553/2295] added CODEOWNERS --- .github/CODEOWNERS | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..636853854 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,20 @@ +# +# Author: Hari Sekhon +# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam + +#* @harisekhon From 17475b8329b4be392ed748bd9014ecdff5b77392 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 11:12:46 +0000 Subject: [PATCH 1554/2295] added fork-update.yaml --- .github/workflows/fork-update.yaml | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/fork-update.yaml diff --git a/.github/workflows/fork-update.yaml b/.github/workflows/fork-update.yaml new file mode 100644 index 000000000..7239ea515 --- /dev/null +++ b/.github/workflows/fork-update.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Update + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + - cron: '0 10 * * 2' + +jobs: + fork_update: + name: Fork Update + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write + pull-requests: write From d54611586c5524480da75511b609da6d3bf637ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:09 +0000 Subject: [PATCH 1555/2295] added fork-sync.yaml --- .github/workflows/fork-sync.yaml | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/fork-sync.yaml diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml new file mode 100644 index 000000000..aa17a532e --- /dev/null +++ b/.github/workflows/fork-sync.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Sync + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + - cron: '0 * * * *' + +jobs: + fork_sync: + if: github.repository_owner != 'HariSekhon' + name: Fork Sync + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write From 865e37884ce7472f216a015fcd650811ed28560b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:10 +0000 Subject: [PATCH 1556/2295] added fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/fork-update-pr.yaml diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml new file mode 100644 index 000000000..66315c238 --- /dev/null +++ b/.github/workflows/fork-update-pr.yaml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Update PR + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + # fork-sync happens for default branch every hour, so just after that, run PRs for branches + - cron: '2 10 * * 2' + +jobs: + fork_update_pr: + if: github.repository_owner != 'HariSekhon' + name: Fork Update PR + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write + pull-requests: write From 29b997ba45497e1726f6497ebfbbe56494e07490 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:37 +0000 Subject: [PATCH 1557/2295] removed .github/workflows/fork-update.yaml --- .github/workflows/fork-update.yaml | 34 ------------------------------ 1 file changed, 34 deletions(-) delete mode 100644 .github/workflows/fork-update.yaml diff --git a/.github/workflows/fork-update.yaml b/.github/workflows/fork-update.yaml deleted file mode 100644 index 7239ea515..000000000 --- a/.github/workflows/fork-update.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Author: Hari Sekhon -# Date: Tue Feb 4 09:53:28 2020 +0000 -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback -# -# https://www.linkedin.com/in/harisekhon -# - ---- -name: Fork Update - -on: - workflow_dispatch: - inputs: - debug: - type: string - required: false - schedule: - - cron: '0 10 * * 2' - -jobs: - fork_update: - name: Fork Update - uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update.yaml@master - with: - debug: ${{ github.event.inputs.debug }} - permissions: - contents: write - pull-requests: write From 0215861686be3ee86ae89b6ca2d4de8b087b424f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:49 +0000 Subject: [PATCH 1558/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a92204784..d15d6b66b 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: From a7039159e2839a128fcfc223f59c1efadffe54d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 1559/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 4b0cb4170..bbcd13ce3 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Alpine 3 uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: From 4124f8dc89b2e5d09edbee7a8a7592feaae3d20e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 1560/2295] updated centos.yaml --- .github/workflows/centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ff1e33598..050cf439d 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 4f207ced80d45d2bfa8c8bbcde54c26d35fa9b86 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 1561/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 588f5de59..876a9b969 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 2a28ec495b2ce83011962023f60efb5fc11edcfd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 1562/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 23aeb1c79..a5ec3ec2a 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 0ffb0b317883095ab4851936beba920394ac953c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 1563/2295] updated checkov.yaml --- .github/workflows/checkov.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index a31ba133a..acb4d2f98 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -37,6 +37,7 @@ on: jobs: checkov: + if: github.repository_owner == 'HariSekhon' name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master permissions: From bbf6a1fc07cadd63dff00f7a1674bf8822abd758 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 1564/2295] updated debian.yaml --- .github/workflows/debian.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 0ac7e1f3e..80950d765 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 553b673faaf1d44fd1eef2f89dcac8174f572e67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 1565/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 7763b32f2..bc2d060e1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 10 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From d53f5159a9951fae92799854f6e4ac7eb5b4071d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 1566/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 8d043b023..426904fbe 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 8 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 36d86213770922e4fd401edf45418537893ccff6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 1567/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6a9c34723..490f89be6 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 9 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From fd931e99636e5b1baa003acedb4a9112cf060ebc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 1568/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e71685633..9fa298875 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e71685633a9bf9d8aa19dc9f9597194e4ce5eae4 +Subproject commit 9fa298875fe72586ff61dc2816fb0bc2b0683f93 From 75d5e1f01f342a1199338c3ed8b6eded844ed60a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 1569/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4fa30554f..aaf5fd06b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4fa30554fc746d5eb90e374c7782bc1bef7f6c47 +Subproject commit aaf5fd06bfa092b87cfbe02557a39678177f8ef6 From a3ebc23158feb58e7210dc97548dd839319122c9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 1570/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 103a1543f..5b548e01b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 103a1543fa73a0adf4f0af710fcf04d099500170 +Subproject commit 5b548e01bb60dd9a3660e5934f9930f4de10a3b5 From 3b4ea49f097b0dffc89dd0b1cf894cb0c1a5b6a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 1571/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9b14e6b9b..2e4052f28 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9b14e6b9be41fd29ba76a585bd490721be05b2d6 +Subproject commit 2e4052f282510229f388b3a9b589373a11cf1afd From 5449c6c9b5be5626ccf61e8bd326990d878c73e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 1572/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 99015df89..e00e7a596 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 6 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 340211a6bf77974c55f5a79b1431be4fc2c73c60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 1573/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 3dea635f2..9b43ff3d6 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 7 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From cc5f606baf71839ae1ce02616aaf5751c002339a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 1574/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index c893bc2cf..605ea0075 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Fedora uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master #with: From 73963c1c732510de80c034b76200e707459ee3ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 1575/2295] updated mac.yaml --- .github/workflows/mac.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ea6c12828..9a815ad32 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: From 10ef68716bc728022dd80874eb1e9fb19675da92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 1576/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index b5c50bc4c..89cb7efbc 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Mac 10.15 uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: From 1e0b033015eadc849368c40939e01c6a5ff8e1c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 1577/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 489d1bcc4..a680ff327 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -27,6 +27,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 47bb0171e97ff9de9d50db4fd03c6276c5665255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 1578/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 5c0e12095..451646399 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -27,6 +27,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 72934f8236afea4a8b83cc739c6189865650d1f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 1579/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c58d3aaa9..6915ea644 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 27e1b6e85389944c97f5868bbdd40d9566c36111 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 1580/2295] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 659bbc6f6..c0e6338f9 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.5 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From bdaec41a4b9235efc09f87211233136a5fbf8eb7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 1581/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b325c9523..697177f9d 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -28,10 +28,9 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master - #with: - # debug: 1 with: version: 3.6 #debug: 1 From 091212ddf8c2a0f7e8bdc59e4d8586fd27035a72 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 1582/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 413e03916..1af8b107b 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From f606e8fcb39eec9ab7e53fce4dda05664640407e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 1583/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 9621587bb..84178955e 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 2cd4e2861042109654a16fc27f1130efa89627a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 1584/2295] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index d18460b45..2de0d75c4 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -35,6 +35,7 @@ on: jobs: semgrep: + if: github.repository_owner == 'HariSekhon' name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: From 84a4076e254cf64e3b992e172d9258c010884e94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 1585/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index af3635d78..692b735f5 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,6 +37,7 @@ on: jobs: semgrep: + if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: From 76c380520b2ab92c42cbddeab9bf74aa7cf97cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 1586/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 07c4af459..5a803936b 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 63cb1babaf2aacfd5e8c9a40b4dca6c1e1acf6fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 1587/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 561d7cfda..40f27c66a 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 14.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 2cec6d28290145168f4e9848cc6673c0c9caa2a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 1588/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 59d45e0ae..d5a04dc36 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 16.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 685a4ed5ad26bcbc37e00ea81248067f506f99e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 1589/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d3818d27d..da34f3b15 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 18.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 1e85ac81707ac26e979c576eee257ace838b319d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 1590/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 7ceee32d7..589268833 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 20.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 5501698916ff88f0b73642037689c1bca20f0729 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:22 +0000 Subject: [PATCH 1591/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 4c463496a..c136b4300 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: GitHub Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: From 4cd5d13d059bdc212433041431f127822d55cab9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 1592/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9fa298875..ddc454546 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9fa298875fe72586ff61dc2816fb0bc2b0683f93 +Subproject commit ddc4545469696c0f6e85370312bf95262c720764 From 04c5f7896e45c77f4a05f6af7e5b43f459924ee0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 1593/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aaf5fd06b..5fa1537ed 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aaf5fd06bfa092b87cfbe02557a39678177f8ef6 +Subproject commit 5fa1537ed9358b5bba786c80826d3dd71485bd39 From 26be37c87b3019e7cf3b3eed19361f22948de81b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 1594/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 5b548e01b..93f740043 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 5b548e01bb60dd9a3660e5934f9930f4de10a3b5 +Subproject commit 93f740043fc57911b4927172f98c7e74f6d1916f From d5a9d19af5fc728d7176b925e87f943c5c180449 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 1595/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2e4052f28..02a6260ff 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2e4052f282510229f388b3a9b589373a11cf1afd +Subproject commit 02a6260ff23b207e9d42506a2b016b9093243018 From b3ec6524b4c0f5469f1e066f07a081e0bde4e75b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 18:51:04 +0000 Subject: [PATCH 1596/2295] switched to more generic disabling repository forks --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index d15d6b66b..0e8afe880 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index bbcd13ce3..0f9112eb1 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Alpine 3 uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 050cf439d..9e3f071e7 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 876a9b969..275c52047 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index a5ec3ec2a..4fa2d3a34 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index acb4d2f98..c1a53e190 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -37,7 +37,7 @@ on: jobs: checkov: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master permissions: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 80950d765..827ba4458 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index bc2d060e1..c251bb27b 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 10 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 426904fbe..30b677c31 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 8 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 490f89be6..4eb3965b2 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 9 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 605ea0075..1ebb473dc 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Fedora uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master #with: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 9a815ad32..eac5b8d7d 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 89cb7efbc..840fc38c3 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Mac 10.15 uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index a680ff327..e3631d1e9 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -27,7 +27,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 451646399..a8fba700c 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -27,7 +27,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 6915ea644..9a73aedad 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index c0e6338f9..5797d5469 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.5 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 697177f9d..d561a026c 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 1af8b107b..2bef6faf8 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 84178955e..c1432111c 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 2de0d75c4..9437a66a8 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -35,7 +35,7 @@ on: jobs: semgrep: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 692b735f5..216ce7971 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,7 +37,7 @@ on: jobs: semgrep: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5a803936b..9139cdf3a 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 40f27c66a..0ecd25dcd 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 14.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5a04dc36..fca6a638d 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 16.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index da34f3b15..47f30eb7b 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 18.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 589268833..5343bf74d 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 20.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index c136b4300..6d938de27 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: GitHub Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: From 38146c5d2827314d7b9a1d9e3caf3e2f4e4c9045 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:24:00 +0000 Subject: [PATCH 1597/2295] updated workflows --- .github/workflows/alpine.yaml | 4 ++-- .github/workflows/alpine_3.yaml | 4 ++-- .github/workflows/centos.yaml | 4 ++-- .github/workflows/centos7.yaml | 4 ++-- .github/workflows/centos8.yaml | 4 ++-- .github/workflows/checkov.yaml | 4 ++-- .github/workflows/debian.yaml | 4 ++-- .github/workflows/debian_10.yaml | 4 ++-- .github/workflows/debian_6.yaml.disabled | 6 +++--- .github/workflows/debian_7.yaml.disabled | 6 +++--- .github/workflows/debian_8.yaml | 4 ++-- .github/workflows/debian_9.yaml | 4 ++-- .github/workflows/fedora.yaml | 4 ++-- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/mac.yaml | 4 ++-- .github/workflows/mac_10.15.yaml | 4 ++-- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 4 ++-- .github/workflows/semgrep.yaml | 4 ++-- .github/workflows/ubuntu.yaml | 4 ++-- .github/workflows/ubuntu_14.04.yaml | 4 ++-- .github/workflows/ubuntu_16.04.yaml | 4 ++-- .github/workflows/ubuntu_18.04.yaml | 4 ++-- .github/workflows/ubuntu_20.04.yaml | 4 ++-- .github/workflows/ubuntu_github.yaml | 4 ++-- 32 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 0e8afe880..bf4134967 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 0f9112eb1..c6de94896 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 9e3f071e7..8aae78cc4 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 275c52047..5ae6adf3d 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4fa2d3a34..3fddfa1c0 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index c1a53e190..12bcbb362 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 827ba4458..d022f6947 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index c251bb27b..fbf1f1c8d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index e00e7a596..1f09cf7b4 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 6 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 9b43ff3d6..2f5e67683 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 7 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 30b677c31..92410bbb5 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 4eb3965b2..6cb201a1e 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 1ebb473dc..5cae17abe 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index aa17a532e..6d6501c3c 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 66315c238..d3f47e00a 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index eac5b8d7d..83fd21b69 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 840fc38c3..c982de7ec 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index e3631d1e9..dd1e67dd8 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # name: PyPy 2 diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index a8fba700c..3ab09c687 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # name: PyPy 3 diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 9a73aedad..76ddb6c87 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 5797d5469..cccd0f02e 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d561a026c..7f6e22b37 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 2bef6faf8..12c884d26 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index c1432111c..64940428c 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 9437a66a8..50d6a29ce 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 216ce7971..0de705b2d 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 9139cdf3a..ccbf75603 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 0ecd25dcd..a3e08b941 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index fca6a638d..f1ebcd56b 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 47f30eb7b..d412b867f 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5343bf74d..56b89774a 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 6d938de27..74a224dc3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- From ab151897c3f141533988778a727858353a7c5199 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 1598/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ddc454546..706664f7b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ddc4545469696c0f6e85370312bf95262c720764 +Subproject commit 706664f7bb88c512bdada2b9480496238896ac35 From 24f6d722c85d5e135445944b78956df9c2ea3378 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 1599/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5fa1537ed..77cf6f982 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5fa1537ed9358b5bba786c80826d3dd71485bd39 +Subproject commit 77cf6f9821c1056c890e845d2044c634f0f6903c From dc49b29e5266a97ef4df5517179d38b646b01171 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 1600/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 93f740043..1774fc022 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 93f740043fc57911b4927172f98c7e74f6d1916f +Subproject commit 1774fc022907673f5c2690bab98dff2e6bbadc50 From c542dcc0745bae6000e4715de53f8e0a88e9a476 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 1601/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 02a6260ff..2118def3c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 02a6260ff23b207e9d42506a2b016b9093243018 +Subproject commit 2118def3c6c352018d9e630753314397bf9a5511 From 8d5cb4da7623b7d905f17f62fab589f1e4a4e97b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:33:14 +0000 Subject: [PATCH 1602/2295] switched to make reusable workflow --- .github/workflows/alpine.yaml | 8 ++++---- .github/workflows/alpine_3.yaml | 8 ++++---- .github/workflows/centos.yaml | 8 ++++---- .github/workflows/centos7.yaml | 8 ++++---- .github/workflows/centos8.yaml | 8 ++++---- .github/workflows/debian.yaml | 8 ++++---- .github/workflows/debian_10.yaml | 8 ++++---- .github/workflows/debian_6.yaml.disabled | 8 ++++---- .github/workflows/debian_7.yaml.disabled | 8 ++++---- .github/workflows/debian_8.yaml | 8 ++++---- .github/workflows/debian_9.yaml | 8 ++++---- .github/workflows/fedora.yaml | 11 ++++++----- .github/workflows/ubuntu.yaml | 8 ++++---- .github/workflows/ubuntu_14.04.yaml | 8 ++++---- .github/workflows/ubuntu_16.04.yaml | 8 ++++---- .github/workflows/ubuntu_18.04.yaml | 8 ++++---- .github/workflows/ubuntu_20.04.yaml | 8 ++++---- .github/workflows/ubuntu_github.yaml | 4 ++-- 18 files changed, 72 insertions(+), 71 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index bf4134967..e1ccb2f59 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: alpine:latest #debug: 1 diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index c6de94896..d09654eca 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3 + container: alpine:3 #debug: 1 diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8aae78cc4..5c7547a49 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: centos:latest #debug: 1 diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5ae6adf3d..8c4dc9246 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 + container: centos:7 #debug: 1 diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3fddfa1c0..910f3728d 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 + container: centos:8 #debug: 1 diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d022f6947..e8b5b2cdb 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: debian:latest #debug: 1 diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index fbf1f1c8d..523a7487f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10 + container: debian:10 #debug: 1 diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1f09cf7b4..1192fe4f7 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 6 + container: debian:6 #debug: 1 diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 2f5e67683..037d7b786 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 + container: debian:7 #debug: 1 diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 92410bbb5..514676132 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 + container: debian:8 #debug: 1 diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6cb201a1e..3e2bbcbca 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 9 + container: debian:9 #debug: 1 diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 5cae17abe..91ff894ce 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,9 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Fedora - uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master - #with: - # debug: 1 + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: fedora + #debug: 1 diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ccbf75603..9aad63ce7 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: ubuntu:latest #debug: 1 diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index a3e08b941..b586ad6aa 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 14.04 + container: ubuntu:14.04 #debug: 1 diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f1ebcd56b..b021f2b27 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 16.04 + container: ubuntu:16.04 #debug: 1 diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d412b867f..c56f3f0af 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 18.04 + container: ubuntu:18.04 #debug: 1 diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56b89774a..8812e844c 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 20.04 + container: ubuntu:20.04 #debug: 1 diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 74a224dc3..7d7d473f3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -27,9 +27,9 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: GitHub Ubuntu + name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: # debug: 1 From a8c3f391a7a59b8c58a37bda8f8ad093814dc8e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 1603/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 706664f7b..ce2cb27e3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 706664f7bb88c512bdada2b9480496238896ac35 +Subproject commit ce2cb27e36aec14f01a89269793a7311d9e3588b From 688101213e83fdc174406851f82a1b5e40c557fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 1604/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 77cf6f982..970d9f9fc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 77cf6f9821c1056c890e845d2044c634f0f6903c +Subproject commit 970d9f9fc46d4d0c45952f83f7a0c53b5a959269 From 67653e6f7f59de7767a0117bb6dde88720c83be3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 1605/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2118def3c..e0212e355 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2118def3c6c352018d9e630753314397bf9a5511 +Subproject commit e0212e35502da33ecfeb2be88513c9053a6eab78 From aad5ba39ca788874abce90d692a3e0e005b29f2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:32 +0000 Subject: [PATCH 1606/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ce2cb27e3..1db94d597 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ce2cb27e36aec14f01a89269793a7311d9e3588b +Subproject commit 1db94d597a27f6afe7c74e1d9c83ae3102c94bf7 From 5ff3247784627ea858d93284e7b1240b45d04405 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:32 +0000 Subject: [PATCH 1607/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 970d9f9fc..d86ca01bc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 970d9f9fc46d4d0c45952f83f7a0c53b5a959269 +Subproject commit d86ca01bcfc1f293310bc956c167fdb0c7dc6a2e From 635e14f3f3bd5a6096164fbde553a30d0e9869a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:33 +0000 Subject: [PATCH 1608/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e0212e355..2d710885e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e0212e35502da33ecfeb2be88513c9053a6eab78 +Subproject commit 2d710885e8c010f4de5e852cd91b0be9fc24e3f3 From fa8c87b916392eebe1c72780551025e5d79268cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:54:19 +0000 Subject: [PATCH 1609/2295] added yaml.yaml --- .github/workflows/yaml.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/yaml.yaml diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml new file mode 100644 index 000000000..e939446a6 --- /dev/null +++ b/.github/workflows/yaml.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: YAML + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + check_yaml: + if: github.event.repository.fork == false + name: Check YAML + uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master From 12646de550ba296cf26e4a058e9a1cc1cc723c8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:57:45 +0000 Subject: [PATCH 1610/2295] added json.yaml --- .github/workflows/json.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/json.yaml diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml new file mode 100644 index 000000000..d42ced6d8 --- /dev/null +++ b/.github/workflows/json.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: JSON + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + check_json: + if: github.event.repository.fork == false + name: Check JSON + uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master From 6f5a551e7b3328c53ddb3653998c3646fe7b7f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 16:27:43 +0000 Subject: [PATCH 1611/2295] added .checkov.yaml --- .checkov.yaml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .checkov.yaml diff --git a/.checkov.yaml b/.checkov.yaml new file mode 100644 index 000000000..8962e2283 --- /dev/null +++ b/.checkov.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C h e c k o v c o n f i g +# ============================================================================ # + +# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file +# +# This is not well documented but the fields seem to be the same as: +# +# checkov --help +# +# See master template at: +# +# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml + +--- +compact: true +directory: + - . +skip-path: + - bash-tools + - pylib + - sql + - templates +docker-image: harisekhon/pytools +download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n' +framework: + - all +no-guide: true +output: cli +quiet: true +repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools' +skip-suppressions: true +soft-fail: true From c7467b25b97c47b926929be4d804120d5349b5ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 18:41:44 +0000 Subject: [PATCH 1612/2295] added .editorconfig --- .editorconfig | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..268166e72 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,78 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# http://EditorConfig.org + +# stop recursing upwards for other .editorconfig files +root = true + +# Unix-style newlines with a newline ending every file +[*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[.*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +# ============================================================================ # +# Older Stuff, don't think I use this any more +# ============================================================================ # + +# Matches multiple files with brace expansion notation +# Set default charset +#[*.{js,py}] +#charset = utf-8 + +# Indentation override for all JS under lib directory +#[lib/**.js] +#indent_style = space +#indent_size = 2 + +# Matches the exact files either package.json or .travis.yml +#[{package.json,.travis.yml}] +#indent_style = space +#indent_size = 2 + +#[*.xml] +#indent_style = space +#indent_size = 2 From bd70bc0208c6ad538db66d2bebfc8a712164ff67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 19:01:49 +0000 Subject: [PATCH 1613/2295] added debug passthrough --- .github/workflows/alpine.yaml | 15 ++++++++++----- .github/workflows/alpine_3.yaml | 15 ++++++++++----- .github/workflows/centos.yaml | 15 ++++++++++----- .github/workflows/centos7.yaml | 15 ++++++++++----- .github/workflows/centos8.yaml | 15 ++++++++++----- .github/workflows/checkov.yaml | 5 +++++ .github/workflows/debian.yaml | 15 ++++++++++----- .github/workflows/debian_10.yaml | 15 ++++++++++----- .github/workflows/debian_6.yaml.disabled | 15 ++++++++++----- .github/workflows/debian_7.yaml.disabled | 15 ++++++++++----- .github/workflows/debian_8.yaml | 15 ++++++++++----- .github/workflows/debian_9.yaml | 15 ++++++++++----- .github/workflows/fedora.yaml | 16 +++++++++++----- .github/workflows/fork-sync.yaml | 3 ++- .github/workflows/fork-update-pr.yaml | 3 ++- .github/workflows/mac.yaml | 7 ++++++- .github/workflows/mac_10.15.yaml | 7 ++++++- .github/workflows/pypy2.yaml | 7 ++++++- .github/workflows/pypy3.yaml | 7 ++++++- .github/workflows/python2.7.yaml | 7 ++++++- .github/workflows/python3.5.yaml | 7 ++++++- .github/workflows/python3.6.yaml | 7 ++++++- .github/workflows/python3.7.yaml | 7 ++++++- .github/workflows/python3.8.yaml | 7 ++++++- .github/workflows/semgrep-cloud.yaml | 5 +++++ .github/workflows/semgrep.yaml | 5 +++++ .github/workflows/ubuntu.yaml | 15 ++++++++++----- .github/workflows/ubuntu_14.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_16.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_18.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_20.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_github.yaml | 13 +++++++++---- 32 files changed, 253 insertions(+), 100 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index bf4134967..659a61612 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: alpine:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index c6de94896..26088e8ec 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3 - #debug: 1 + container: alpine:3 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8aae78cc4..901a90e9f 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: centos:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5ae6adf3d..6e3ebb0e6 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 - #debug: 1 + container: centos:7 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3fddfa1c0..e782ababf 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 - #debug: 1 + container: centos:8 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 12bcbb362..85e629364 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -32,6 +32,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d022f6947..ed706c124 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: debian:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index fbf1f1c8d..e2aa2395d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10 - #debug: 1 + container: debian:10 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1f09cf7b4..c04220d48 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 6 - #debug: 1 + container: debian:6 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 2f5e67683..26f23cce1 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 - #debug: 1 + container: debian:7 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 92410bbb5..d551a85c0 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 - #debug: 1 + container: debian:8 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6cb201a1e..d92ec4607 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 9 - #debug: 1 + container: debian:9 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 5cae17abe..a83cad0c7 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,9 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Fedora - uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master - #with: - # debug: 1 + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: fedora + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 6d6501c3c..98331b7d8 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -18,8 +18,9 @@ on: workflow_dispatch: inputs: debug: - type: string + type: boolean required: false + default: false schedule: - cron: '0 * * * *' diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index d3f47e00a..d3b156e38 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -18,8 +18,9 @@ on: workflow_dispatch: inputs: debug: - type: string + type: boolean required: false + default: false schedule: # fork-sync happens for default branch every hour, so just after that, run PRs for branches - cron: '2 10 * * 2' diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 83fd21b69..09049616c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: latest - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index c982de7ec..8f328ca80 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: 10.15 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index dd1e67dd8..a625697b6 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -18,6 +18,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -32,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 3ab09c687..c0ba3825e 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -18,6 +18,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -32,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 76ddb6c87..beca7bcda 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index cccd0f02e..fa0f7ddbf 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.5 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 7f6e22b37..5639271ab 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 12c884d26..25a455a85 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 64940428c..095e991ac 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 50d6a29ce..0b7e88f01 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -30,6 +30,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 0de705b2d..ecb59b85f 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -32,6 +32,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ccbf75603..ff42db345 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: ubuntu:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index a3e08b941..53a46cf08 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 14.04 - #debug: 1 + container: ubuntu:14.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f1ebcd56b..44c7399e7 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 16.04 - #debug: 1 + container: ubuntu:16.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d412b867f..e8aca96c3 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 18.04 - #debug: 1 + container: ubuntu:18.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56b89774a..f5e017c96 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 20.04 - #debug: 1 + container: ubuntu:20.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 74a224dc3..f6a2a69f4 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,9 +32,9 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: GitHub Ubuntu + name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master - #with: - # debug: 1 + with: + # debug: true From f8ba2a996acadf6c110c383806f218a1f73d8851 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 19:53:56 +0000 Subject: [PATCH 1614/2295] moved permissions key to top and added missing concurrency build auto-cancellations --- .github/workflows/checkov.yaml | 11 +++++++---- .github/workflows/fork-sync.yaml | 9 +++++++-- .github/workflows/fork-update-pr.yaml | 11 ++++++++--- .github/workflows/semgrep-cloud.yaml | 4 ++++ .github/workflows/semgrep.yaml | 13 +++++++++---- .github/workflows/ubuntu_github.yaml | 2 +- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 85e629364..d05b53148 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -40,12 +40,15 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + actions: read + contents: read + security-events: write + jobs: checkov: if: github.event.repository.fork == false name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master - permissions: - actions: read - contents: read - security-events: write + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 98331b7d8..fd91109e2 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -24,6 +24,13 @@ on: schedule: - cron: '0 * * * *' +permissions: + contents: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false + jobs: fork_sync: if: github.repository_owner != 'HariSekhon' @@ -31,5 +38,3 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: debug: ${{ github.event.inputs.debug }} - permissions: - contents: write diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index d3b156e38..12449fb36 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -25,6 +25,14 @@ on: # fork-sync happens for default branch every hour, so just after that, run PRs for branches - cron: '2 10 * * 2' +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false + jobs: fork_update_pr: if: github.repository_owner != 'HariSekhon' @@ -32,6 +40,3 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: debug: ${{ github.event.inputs.debug }} - permissions: - contents: write - pull-requests: write diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 0b7e88f01..da4ccbdbd 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -38,6 +38,10 @@ on: schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: semgrep: if: github.event.repository.fork == false diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index ecb59b85f..410e63c0f 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -40,12 +40,17 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: semgrep: if: github.event.repository.fork == false name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permissions: - actions: read - contents: read - security-events: write diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index f6a2a69f4..83a9e46c8 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,4 +37,4 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master with: - # debug: true + debug: ${{ github.event.inputs.debug }} From bbf552ad3fbf71333816f360c1c55daaaada8640 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 1615/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1db94d597..57ea4e7cf 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1db94d597a27f6afe7c74e1d9c83ae3102c94bf7 +Subproject commit 57ea4e7cf483d6faecf630bc0b1173f80ab127d4 From acc023e945f335442b2256472144cc9983e05802 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 1616/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d86ca01bc..eae777b99 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d86ca01bcfc1f293310bc956c167fdb0c7dc6a2e +Subproject commit eae777b99fb25d19587b0c0f3d22c1f838ecf169 From 132b991b108146fd9813f212229aa409e3c86c08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 1617/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 1774fc022..dc6c1cd42 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 1774fc022907673f5c2690bab98dff2e6bbadc50 +Subproject commit dc6c1cd4278bda75c73dad895daa72846f4ff8e9 From b118ae872145afa0b07ad8dd5814ab11ca80e145 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 1618/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d710885e..1afa44055 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d710885e8c010f4de5e852cd91b0be9fc24e3f3 +Subproject commit 1afa44055e21a2ec8631f298e13898e7f1c8bdc3 From 72b56ece611667531b519adcf600f52a97395912 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Feb 2022 00:01:01 +0000 Subject: [PATCH 1619/2295] tweaked debug pass through as github.event.inputs.debug isn't present in scheduled runs --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/json.yaml | 5 +++++ .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/yaml.yaml | 9 +++++++++ 32 files changed, 44 insertions(+), 30 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 659a61612..21aacec90 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 26088e8ec..6aaab2a03 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 901a90e9f..1c611d673 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 6e3ebb0e6..4454ab3d9 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index e782ababf..4ba82161d 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index d05b53148..2e57716ad 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,4 +51,4 @@ jobs: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index ed706c124..a733a4647 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e2aa2395d..813fbd9f9 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index c04220d48..94a1d5176 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 26f23cce1..5ccf6bb63 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index d551a85c0..613e1e15e 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index d92ec4607..398709d88 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index a83cad0c7..49924b3fb 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index fd91109e2..187feb41e 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -37,4 +37,4 @@ jobs: name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 12449fb36..e4d89e440 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -39,4 +39,4 @@ jobs: name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index d42ced6d8..4dab279cc 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -24,6 +24,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 09049616c..08766ad90 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 8f328ca80..4d0316349 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: 10.15 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index a625697b6..cb0cfc3af 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -37,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index c0ba3825e..fdfc2d794 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -37,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index beca7bcda..e1d3fced4 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index fa0f7ddbf..5ebb24d18 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.5 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 5639271ab..60a3b9840 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 25a455a85..093d7d27b 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 095e991ac..c925e6db7 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ff42db345..9deae95db 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 53a46cf08..be93c352b 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 44c7399e7..d5c3a1924 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index e8aca96c3..b363c2a92 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index f5e017c96..f0d393ffc 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 83a9e46c8..0f6866651 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,4 +37,4 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index e939446a6..d416105c6 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -24,9 +24,18 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: check_yaml: if: github.event.repository.fork == false From be079535eb00b469161bc52bf3345730edd31d63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Feb 2022 23:37:09 +0000 Subject: [PATCH 1620/2295] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 187feb41e..f139e9617 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 * * * *' + - cron: '0 9 * * *' permissions: contents: write From 2953d2da34054e172c99dd37bd5d8aad2d7217e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 11:35:07 +0000 Subject: [PATCH 1621/2295] updated validate.yaml --- .github/workflows/validate.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index cf901ed71..0594bda8b 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -24,10 +24,20 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: validate: + if: github.event.repository.fork == false name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From bcbfaa4460c4edf6073325a9203624620d8281ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 14:00:01 +0000 Subject: [PATCH 1622/2295] updated CODEOWNERS --- .github/CODEOWNERS | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 636853854..962ae64c7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,4 +17,10 @@ # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam -#* @harisekhon +# * includes changes under .github/ + +#* @harisekhon +#* @myorg/platform-engineering # team based is the way to go +#* @myorg/devops +#k8s @myorg/devops +#src/* @myorg/developers From 198336c212933ea4d17dad22b54ab9927cf6a586 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 14:50:22 +0000 Subject: [PATCH 1623/2295] updated CODEOWNERS --- .github/CODEOWNERS | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 962ae64c7..ad861088b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,10 +17,22 @@ # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam +# Tips: +# # * includes changes under .github/ +# dir/* only matches first level file changes but doesn't recurse +# dir/ recurses +# +# - CODEOWNERS in base branch of PR determines review request +# - paths are case sensitive +# - last match wins, use * at top for overall owner then override with more specific teams -#* @harisekhon -#* @myorg/platform-engineering # team based is the way to go -#* @myorg/devops -#k8s @myorg/devops -#src/* @myorg/developers +#* @harisekhon # username or email address +#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access +#* @myorg/devops +#k8s @myorg/devops @myorg/sre-team +#apps/ @myorg/developers +#apps/dir2 # ignores dir2 as no owner/team specified on this line +#src/ @myorg/developers +#docs/ docs@example.com +#.github/workflows @ci-cd-team From 57d47f1ed5133cef5d9f04abdfa22fcd34404e99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:29:47 +0000 Subject: [PATCH 1624/2295] added codeowners.yaml --- .github/workflows/codeowners.yaml | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/codeowners.yaml diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml new file mode 100644 index 000000000..c0ae12715 --- /dev/null +++ b/.github/workflows/codeowners.yaml @@ -0,0 +1,43 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Codeowners + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + validate: + if: github.event.repository.fork == false + name: Validate CODEOWNERS + uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master From ed2d55fb11c9fc94dd622d3958ce957c50af1360 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 1625/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 57ea4e7cf..fcd7360af 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 57ea4e7cf483d6faecf630bc0b1173f80ab127d4 +Subproject commit fcd7360af7320590dcae536dafabfbfa3afae11e From 899248b936162e84d0fa5035838830bf42ce07d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 1626/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eae777b99..6e7b0b552 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eae777b99fb25d19587b0c0f3d22c1f838ecf169 +Subproject commit 6e7b0b552fd5ef9d0ad885f9909c6d8e5b765d2f From 14dc0fe8aabb0353b9f4c12436adc0b145b50df9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 1627/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index dc6c1cd42..0ca10f23b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit dc6c1cd4278bda75c73dad895daa72846f4ff8e9 +Subproject commit 0ca10f23bde76e0533c5dc85cfc325dd641de9ee From 8339dc2c903550a25907181ea8cbc1344a90398d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 1628/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 1afa44055..c45cb02b5 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 1afa44055e21a2ec8631f298e13898e7f1c8bdc3 +Subproject commit c45cb02b56adec293d593ea501cf51821a24c090 From dacbcedd624f659676edbccb952c539e636adf51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:45:30 +0000 Subject: [PATCH 1629/2295] renamed .github/workflows/python3.5.yaml to .github/workflows/python3.10.yaml --- .github/workflows/{python3.5.yaml => python3.10.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.5.yaml => python3.10.yaml} (100%) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.10.yaml similarity index 100% rename from .github/workflows/python3.5.yaml rename to .github/workflows/python3.10.yaml From 028eca92ea47ec8f2ebdccba6c7ff160c53497da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:45:54 +0000 Subject: [PATCH 1630/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 5ebb24d18..59657673a 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -12,7 +12,7 @@ # --- -name: Python 3.5 +name: Python 3.10 on: push: @@ -34,8 +34,8 @@ concurrency: jobs: build: if: github.event.repository.fork == false - name: Python 3.5 + name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: - version: 3.5 + version: 3.10 debug: ${{ github.event.inputs.debug || false }} From b7960babb4b69af2181d1f24410c12470f7e028c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:47:21 +0000 Subject: [PATCH 1631/2295] added python3.9.yaml --- .github/workflows/python3.9.yaml | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/python3.9.yaml diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml new file mode 100644 index 000000000..f1ae3982c --- /dev/null +++ b/.github/workflows/python3.9.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.9 + +on: + push: + branches: + - master + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + if: github.event.repository.fork == false + name: Python 3.9 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.9 + debug: ${{ github.event.inputs.debug || false }} From afc0fdf8f8d6d3b00e9ea5bc8ceefe8057869abf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:48:10 +0000 Subject: [PATCH 1632/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2445c90db..ae1ff5cdb 100644 --- a/README.md +++ b/README.md @@ -91,10 +91,11 @@ Hari Sekhon - DevOps Python Tools [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) [![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.5](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.5/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.5%22) [![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) +[![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) +[![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From 9d797c6972f3379773e77c9d00f2b6076c9927c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 13:28:18 +0000 Subject: [PATCH 1633/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 0f6866651..906236af4 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -35,6 +35,6 @@ jobs: make: if: github.event.repository.fork == false name: Make - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: debug: ${{ github.event.inputs.debug || false }} From e633e2a6e5f2485886d698b402b801d6b6b8270f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:02 +0000 Subject: [PATCH 1634/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 21aacec90..ed007352a 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest + caches: apk pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5b2b31359a3e1a05e9ba20a67f3cf5a956d47ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1635/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 6aaab2a03..45290c0db 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 + caches: apk pip cpanm debug: ${{ github.event.inputs.debug || false }} From 04e7be1742349530646bed0ad8b494a17b3ba697 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1636/2295] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 1c611d673..9e3306bfc 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1e4ea79fc79a5a0b34274828c1175909ff468636 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1637/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 4454ab3d9..7f116373b 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 39fb995f35275e7c7bb5597bc3e078f302fb61b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1638/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4ba82161d..20dfc35c4 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From a3638521c5c848280b50141cacb50704297ec094 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1639/2295] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index c0ae12715..d4e575f30 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 15c0be92da5ab97a5002f833821ae79611c77ae4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 1640/2295] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index a733a4647..c3fd4fc9b 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From fb319ef5d2c89609c9a55a3f5f159573a7013877 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 1641/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 813fbd9f9..b76dcd103 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From c6655dae1c7a791a247f70c93f3e82e6c7a63807 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 1642/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 94a1d5176..b11a56258 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From e82dfd8d2ed275527c78a0e0a00a3168dfcdfa81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 1643/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 5ccf6bb63..e523af78f 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 364a8dded1efecb9b39d0c75fe597788c07c580a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 1644/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 613e1e15e..fe764801c 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1dd4ddaf2f51b546e784a5ea057e24b7e629f762 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 1645/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 398709d88..f473395da 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 7abd77f2fed7bbaff7c7502c5de8ba3efa2ed796 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 1646/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 49924b3fb..b29f4c229 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 05a4812c47c89ecfda6f4ce3c0600e8cbe0e0975 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 1647/2295] updated json.yaml --- .github/workflows/json.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 4dab279cc..b66e8cee6 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + jobs: check_json: if: github.event.repository.fork == false From 985f82224ea0755d8343c91bce5c3b2956adf4e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 1648/2295] updated mac.yaml --- .github/workflows/mac.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 08766ad90..bbee846e8 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -27,15 +27,19 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Mac - uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + runs-on: macos-latest + caches: brew pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5be50a20225e0ffc971588c2564668c55461fa3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 1649/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 4d0316349..f2f6a57ac 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -27,15 +27,19 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Mac 10.15 - uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10.15 + runs-on: macos-10.15 + caches: brew pip cpanm debug: ${{ github.event.inputs.debug || false }} From 56a1eb7cc5e4b4fc4037c4853ddbe96e0e70bbdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 1650/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index cb0cfc3af..c89f841fa 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 21c36b043001da8af2014039880750cab988ef62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1651/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index fdfc2d794..ecdf83762 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 6361ac5869bed7de5c42a6f20191db8443aadcfd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1652/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index e1d3fced4..53b6b6171 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From accbbef60a0336d5c6bd7ec017d639fce60add68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1653/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 59657673a..3552a5a98 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -37,5 +40,5 @@ jobs: name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: - version: 3.10 + version: "3.10" debug: ${{ github.event.inputs.debug || false }} From cea4bb065d3c595153700cc7af6c1259cc979aef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1654/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 60a3b9840..b3e23f7fa 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From b985dce49d26e14b539918c4e30692e6c7730319 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1655/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 093d7d27b..168caa2e8 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From fc716d57a8a07692fe2952cd5f3e295a6e7e8b7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 1656/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index c925e6db7..b8ae52619 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From d579d1734efff4a3b0ac8d6a9dc8b822deff5d7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 1657/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index f1ae3982c..580830e31 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 9f83f0bbdd25288c15c806a4a50183fab050692b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 1658/2295] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index da4ccbdbd..1e8278f7e 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -38,6 +38,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From b4ec7c3d6393a8e87af2d87e4d56c1186e6c78d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 1659/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 9deae95db..20c11bc5e 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From ac808c2b717941eccdfbdc89645106856bde240b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 1660/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index be93c352b..433a138f5 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From f7651ec3b60ceb6d508e6ac157e30f7dbe87a243 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 1661/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5c3a1924..46daed829 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1c113092e3aa8979b36d043c2383e4914293238f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 1662/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index b363c2a92..b22c72cd9 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From de492e31675c57a903257018e443f6abf07fa8d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 1663/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index f0d393ffc..cc27c8016 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From f89e3967dc0e99ee20748de1ae4e3c2b73bbf920 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 1664/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 906236af4..323631dca 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -37,4 +40,5 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5b9740262c95f478d89fd83655326f18fa4b69d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 1665/2295] updated validate.yaml --- .github/workflows/validate.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 0594bda8b..bbdc02826 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 376e27aee3256e04d38d325a10a20735122dd9f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 1666/2295] updated yaml.yaml --- .github/workflows/yaml.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d416105c6..c11764e11 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 09a740dc794c03556ab0503bf60e8df6b25c7e59 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:47:56 +0000 Subject: [PATCH 1667/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index b11a56258..bb6698b9a 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -41,5 +41,6 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 - caches: apt pip cpanm + # causes nodejs errors + #caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From b42d05f595a680ed4a097a3ef883bcf54e2b27f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:47:57 +0000 Subject: [PATCH 1668/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e523af78f..216adb436 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -41,5 +41,6 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 - caches: apt pip cpanm + # causes nodejs errors + #caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 850f12f5d9a678c6438009a78d54fa68eddd0606 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 1669/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fcd7360af..1f9ee12e5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fcd7360af7320590dcae536dafabfbfa3afae11e +Subproject commit 1f9ee12e52063c59c0620842bfe7186e91bec5a8 From e7c80a8460b1d227ac466d3a41da480505e5f374 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 1670/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e7b0b552..aec17428f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e7b0b552fd5ef9d0ad885f9909c6d8e5b765d2f +Subproject commit aec17428f1a946d4b2ead6b37379b84c9b625a33 From b16b3c9c5d249f98c60d01dab8e777443421f93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 1671/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0ca10f23b..69aff25cb 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0ca10f23bde76e0533c5dc85cfc325dd641de9ee +Subproject commit 69aff25cbac90c8fc97a66c0d9e4351ea659c54e From 06840d5c99cc0a788d18705e5498e118dd112712 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 1672/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c45cb02b5..e444f83ec 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c45cb02b56adec293d593ea501cf51821a24c090 +Subproject commit e444f83ecb0082c0639c4df23ca24e7790955d7b From 34e75fb374570061730abff675e28baac361566c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Mar 2022 19:04:09 +0000 Subject: [PATCH 1673/2295] updated .pylintrc --- .pylintrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 65e97ab22..cd3e053f6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -59,7 +59,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111 +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string [REPORTS] @@ -108,7 +108,7 @@ name-group= include-naming-hint=no # Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ +function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ # Naming hint for function names function-name-hint=[a-z_][a-z0-9_]{2,30}$ From 629beaff5a425be174916121208bc981d50ceb3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Mar 2022 19:08:08 +0000 Subject: [PATCH 1674/2295] updated .pylintrc --- .pylintrc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index cd3e053f6..c20ac8f94 100644 --- a/.pylintrc +++ b/.pylintrc @@ -274,7 +274,15 @@ ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). This supports can work # with qualified names. -ignored-classes= +# +# avoids the following error: +# +# pylint -E ./check_zookeeper_version.py +# +# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) +# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) +# +ignored-classes=SQLObject,_socketobject # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular From 0a61737c56e004ea6cb0a3a2293b9a5ba7001919 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Mar 2022 18:47:59 +0000 Subject: [PATCH 1675/2295] updated debug passthrough --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index ed007352a..287b2d490 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -42,4 +42,4 @@ jobs: with: container: alpine:latest caches: apk pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 45290c0db..96f31e3ab 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -42,4 +42,4 @@ jobs: with: container: alpine:3 caches: apk pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 9e3306bfc..ed6372ec7 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:latest caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 7f116373b..681be974f 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:7 caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 20dfc35c4..9c4311285 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:8 caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 2e57716ad..d05b53148 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,4 +51,4 @@ jobs: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index c3fd4fc9b..ae2afe46e 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:latest caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b76dcd103..2656f55fd 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:10 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index bb6698b9a..dff981bd2 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -43,4 +43,4 @@ jobs: container: debian:6 # causes nodejs errors #caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 216adb436..74b06b44e 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -43,4 +43,4 @@ jobs: container: debian:7 # causes nodejs errors #caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index fe764801c..6b2a6f979 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:8 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index f473395da..3641c892c 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:9 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index b29f4c229..2128939b8 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -42,4 +42,4 @@ jobs: with: container: fedora caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index f139e9617..556bd80d9 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -37,4 +37,4 @@ jobs: name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index e4d89e440..12449fb36 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -39,4 +39,4 @@ jobs: name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index bbee846e8..81aaec5b1 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -42,4 +42,4 @@ jobs: with: runs-on: macos-latest caches: brew pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index f2f6a57ac..db4781fab 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -42,4 +42,4 @@ jobs: with: runs-on: macos-10.15 caches: brew pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index c89f841fa..ae3774daf 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -40,4 +40,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index ecdf83762..966741006 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -40,4 +40,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 53b6b6171..bb03008d4 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 3552a5a98..4ba2d2cdf 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: "3.10" - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b3e23f7fa..d94daf551 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 168caa2e8..9e2167f91 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index b8ae52619..0adb02bb5 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 580830e31..04a325e29 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.9 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 20c11bc5e..927f7edc2 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:latest caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 433a138f5..fff40eaa6 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:14.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 46daed829..cd96b0858 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:16.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index b22c72cd9..385bcfff6 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:18.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index cc27c8016..360a8db4e 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:20.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 323631dca..59a9c9110 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} From 11a554febd4d3764100e4494217e214e2c5a93c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 12:26:46 +0100 Subject: [PATCH 1676/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae1ff5cdb..b0e453c94 100644 --- a/README.md +++ b/README.md @@ -47,13 +47,13 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) +[![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) From 5b51de257400d01702714b56e5786d2569ca3f55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 12:31:39 +0100 Subject: [PATCH 1677/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0e453c94..ec5ba95b9 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,12 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) +[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) -[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) From 9e32b2e011867d59a73cb650fe887c95fdc4ac7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 1678/2295] renamed dockerhub_pytools_alpine.yaml to docker_pytools_alpine.yaml --- .../{dockerhub_pytools_alpine.yaml => docker_pytools_alpine.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_alpine.yaml => docker_pytools_alpine.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_alpine.yaml rename to .github/workflows/docker_pytools_alpine.yaml From 5155a7898353142de3a40954d1323315d8d5040e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 1679/2295] renamed dockerhub_pytools_centos.yaml to docker_pytools_centos.yaml --- .../{dockerhub_pytools_centos.yaml => docker_pytools_centos.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_centos.yaml => docker_pytools_centos.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_centos.yaml rename to .github/workflows/docker_pytools_centos.yaml From 90d8b86bc867116df6489163321cfe504f2062fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 1680/2295] renamed dockerhub_pytools_debian.yaml to docker_pytools_debian.yaml --- .../{dockerhub_pytools_debian.yaml => docker_pytools_debian.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_debian.yaml => docker_pytools_debian.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_debian.yaml rename to .github/workflows/docker_pytools_debian.yaml From 33a93df4781346aedf83f7ae33fe5cf9ef8f50fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 1681/2295] renamed dockerhub_pytools_fedora.yaml to docker_pytools_fedora.yaml --- .../{dockerhub_pytools_fedora.yaml => docker_pytools_fedora.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_fedora.yaml => docker_pytools_fedora.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_fedora.yaml rename to .github/workflows/docker_pytools_fedora.yaml From 0f8f7a486e261d442b3aa8d1af7338895b77085f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 1682/2295] renamed dockerhub_pytools_ubuntu.yaml to docker_pytools_ubuntu.yaml --- .../{dockerhub_pytools_ubuntu.yaml => docker_pytools_ubuntu.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_ubuntu.yaml => docker_pytools_ubuntu.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_ubuntu.yaml rename to .github/workflows/docker_pytools_ubuntu.yaml From 338a6c8bc3886c233dcbb9ea394fc4349c8f8c2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:25 +0100 Subject: [PATCH 1683/2295] updated docker_pytools_alpine.yaml --- .github/workflows/docker_pytools_alpine.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index ff2197707..c3687b351 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Alpine) +name: Docker Build (Alpine) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: alpine + repo_tags: | + harisekhon/pytools:alpine + ghcr.io/harisekhon/pytools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-alpine secrets: From fe9bed2863a05acd3df3b931a193faa55afb17bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:26 +0100 Subject: [PATCH 1684/2295] updated docker_pytools_centos.yaml --- .github/workflows/docker_pytools_centos.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 06778bd22..92b36e7d5 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (CentOS) +name: Docker Build (CentOS) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: latest centos + repo_tags: | + harisekhon/pytools:centos + ghcr.io/harisekhon/pytools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-centos secrets: From cdfa7e53803c7558f64d9e8932d4a1384120640e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:29 +0100 Subject: [PATCH 1685/2295] updated docker_pytools_debian.yaml --- .github/workflows/docker_pytools_debian.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index aa9da7e6a..8b34df315 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Debian) +name: Docker Build (Debian) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: debian + repo_tags: | + harisekhon/pytools:debian + ghcr.io/harisekhon/pytools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-debian secrets: From e808440babee55333beea4a93dd027c71ed585e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:31 +0100 Subject: [PATCH 1686/2295] updated docker_pytools_fedora.yaml --- .github/workflows/docker_pytools_fedora.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 3aa3c800f..190cf703b 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Fedora) +name: Docker Build (Fedora) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: fedora + repo_tags: | + harisekhon/pytools:fedora + ghcr.io/harisekhon/pytools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-fedora secrets: From 3deb0857ff51081a3eda899d25a93975e095a213 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:32 +0100 Subject: [PATCH 1687/2295] updated docker_pytools_ubuntu.yaml --- .github/workflows/docker_pytools_ubuntu.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index d53930274..fe5e51a95 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Ubuntu) +name: Docker Build (Ubuntu) on: push: @@ -26,8 +26,11 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: ubuntu latest + repo_tags: | + harisekhon/pytools:latest + harisekhon/pytools:ubuntu + ghcr.io/harisekhon/pytools:latest + ghcr.io/harisekhon/pytools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-ubuntu secrets: From da4664c347c765612a4ca40c698f18c191c80231 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:23:37 +0100 Subject: [PATCH 1688/2295] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index ec5ba95b9..908e5d1b1 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ Hari Sekhon - DevOps Python Tools [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) +[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) +[![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) +[![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) +[![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) + [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) [![Mac 10.15](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac%2010.15/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac+10.15%22) From 7dae166c5609f775c8a60e77a1d7fc166d4fc3c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 1689/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1f9ee12e5..4f58bb28f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1f9ee12e52063c59c0620842bfe7186e91bec5a8 +Subproject commit 4f58bb28f8f2d16cadbf921522ccc044d643728b From d1d84c8a4f2e24d005c199aee8768dfdd6d2e005 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 1690/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aec17428f..85ed84469 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aec17428f1a946d4b2ead6b37379b84c9b625a33 +Subproject commit 85ed84469db95f2e66b5f1e3d0a044a80a49885f From 317a75c51d787d24f0a590a550ad5b09a843ae1f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 1691/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 69aff25cb..0c8255344 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 69aff25cbac90c8fc97a66c0d9e4351ea659c54e +Subproject commit 0c825534472969e6a41770e9246ede7861dbbb10 From 89766f027592e0e3d657e37261a6c27abf77b508 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 1692/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e444f83ec..3417faa70 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e444f83ecb0082c0639c4df23ca24e7790955d7b +Subproject commit 3417faa7035c29de864f101c730a01176f563f38 From 9d5f41536225ef8e93b644898119b5347e273b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:27:27 +0100 Subject: [PATCH 1693/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 908e5d1b1..57a606b7b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) --> -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) From cdaa1d3707cb8593421091ac92a0a74cf31c81ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:29 +0100 Subject: [PATCH 1694/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f58bb28f..07871b12f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f58bb28f8f2d16cadbf921522ccc044d643728b +Subproject commit 07871b12f83f5870d7ada82fa3de8b43ed0b56d4 From 945574937e48a2554487e1324c1953b174cde83d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:29 +0100 Subject: [PATCH 1695/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85ed84469..965905dd6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85ed84469db95f2e66b5f1e3d0a044a80a49885f +Subproject commit 965905dd62b8c402f36a546c043fad07d4ae9d92 From 452d52c2f0c6d4650061485e653e1c0a5965a985 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:30 +0100 Subject: [PATCH 1696/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3417faa70..e116df4dd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3417faa7035c29de864f101c730a01176f563f38 +Subproject commit e116df4ddff7d917fa28291cbca1f1744c8bc664 From 22760c05091ab30ac7d394214904e1d1db7a625b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 10:36:57 +0100 Subject: [PATCH 1697/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57a606b7b..b972256b0 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) -[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) +[![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From a26a0cc966cb71d66ecc03d7300e69077e688110 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:12 +0100 Subject: [PATCH 1698/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 287b2d490..a2dfe24e2 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From d183af7d3fe3279cccfe876a19c12268963b5067 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:13 +0100 Subject: [PATCH 1699/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 96f31e3ab..2a7fbaff0 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From b9127c3d25bf64329361d9bddd88adb95c6aaea0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 1700/2295] updated centos.yaml --- .github/workflows/centos.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ed6372ec7..fbaf19c4f 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 56ed0ad957eebdf7931345efe48cec969f552bdf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 1701/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 681be974f..ebaf5fae2 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 007cf727eb4d8bd6b9c1ca30286385675463b061 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 1702/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 9c4311285..fada215de 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From c931a9e1bc77899cc161faea6074940022e52ec8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 1703/2295] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index d05b53148..26be616d6 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -27,10 +27,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 21daa159117d7b16b845fcb2bfe9c2459874a64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 1704/2295] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index d4e575f30..934d09b92 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -19,10 +19,16 @@ on: branches: - master - main + paths: + - CODEOWNERS + - .github/CODEOWNERS pull_request: branches: - master - main + paths: + - CODEOWNERS + - .github/CODEOWNERS workflow_dispatch: inputs: debug: From 7656bda959896e8b68d1b45ba321b518efeb7008 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 1705/2295] updated debian.yaml --- .github/workflows/debian.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index ae2afe46e..e993c7222 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 9a5c9408ed80b2b874c75589e49ea33edab99248 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 1706/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 2656f55fd..b120564e9 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From b8e0debfd788747913737985f936a6b1763127b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 1707/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index dff981bd2..2e8c06e55 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From ddcbd8312c126630c576b86cfc9fdf2302507fb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 1708/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 74b06b44e..65fe55cca 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 8e76ebbe7956dfd59daaebf459ed25f2ea34f40d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 1709/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 6b2a6f979..ae2bc9f9b 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From d4f65ff7955209a09a20185f32b192b00f2f819e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 1710/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 3641c892c..9223474c1 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From a823e30859dbee0eb6d60b61c5c8ccc15940c961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 1711/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 2128939b8..fe6bcbe6f 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 4d496a1955bfa19a95949f22df958272b346ac91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:18 +0100 Subject: [PATCH 1712/2295] updated json.yaml --- .github/workflows/json.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index b66e8cee6..5a9df06b5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -19,10 +19,14 @@ on: branches: - master - main + paths: + - '**/*.json' pull_request: branches: - master - main + paths: + - '**/*.json' workflow_dispatch: inputs: debug: From 19cf93bff84c4f0befe5657cece5530619581e16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:18 +0100 Subject: [PATCH 1713/2295] updated mac.yaml --- .github/workflows/mac.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 81aaec5b1..92e326bcb 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 2bae838d15d705147545257da15ccb89b14815cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:19 +0100 Subject: [PATCH 1714/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index db4781fab..b8feb60c0 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 61d92389ae1a26f11e498a7d765705cd02f4cd43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:19 +0100 Subject: [PATCH 1715/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index ae3774daf..707dd82f0 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -17,6 +17,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 531040f2624195c209434d5652a9ea626d4e933d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 1716/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 966741006..c379246c9 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -17,6 +17,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 08a217b29f5d3cfa77fe45e6f4d0d2016e830ecd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 1717/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index bb03008d4..a850b2b84 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From d1cf2e7c7c99f81af2c791d7ed46d406dbd49be5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 1718/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 4ba2d2cdf..da5b11f35 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 65919da01b1375e3228520ca50a2b645cb3eed26 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 1719/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d94daf551..32e3089e2 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 636e1258db0c350109ec72fa3ba70de681a70f42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 1720/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 9e2167f91..d6efcf883 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 9db851fdc24075bf69f22df58bf862bc20bd13f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 1721/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 0adb02bb5..f0eb8fa2b 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 21f40ffdd707e62456771c7a39509cd41e5e899c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 1722/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 04a325e29..df374c960 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From d4ba19a16ee1346e6b1856cb44acff99da51b5d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 1723/2295] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 1e8278f7e..47fe5fcf5 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -25,10 +25,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 374acc85391a98039f23afd172bf6f6b8fad7128 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 1724/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 410e63c0f..ea33c7f40 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -27,10 +27,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 541276e36f87f19734af2300e0bc98f74a331ff0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 1725/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 927f7edc2..6a623c6da 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From cd2ef8bfae5853d3363be2cc7d5af6ff0423b87a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 1726/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index fff40eaa6..915f04ae9 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From c2f35969f7ee49177cf2b697c532cce0bc811824 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 1727/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index cd96b0858..f459f4398 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From df957eaa9e121ab791f9f61801390b173207998b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:24 +0100 Subject: [PATCH 1728/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 385bcfff6..926c7e0a5 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 8370c6011bcdec7d2cee8c5455ad23dd4a033031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:24 +0100 Subject: [PATCH 1729/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 360a8db4e..ad419846f 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 654d37ca583fa97480860ab36c5b5d7ad2eee017 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 1730/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 59a9c9110..85bb7dd48 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 520778e57013988327c8c298cff3be492e88909d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 1731/2295] updated validate.yaml --- .github/workflows/validate.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index bbdc02826..b9634e4ae 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -19,10 +19,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 5862e20bb9cf0e6c8e3d027a0d721f9f33caa96f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 1732/2295] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index c11764e11..42f1aae0e 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -19,10 +19,16 @@ on: branches: - master - main + paths: + - '**/*.yml' + - '**/*.yaml' pull_request: branches: - master - main + paths: + - '**/*.yml' + - '**/*.yaml' workflow_dispatch: inputs: debug: From cad5cf8bef2f789d86619ee6ea6c2c52fd1baa3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:26:38 +0100 Subject: [PATCH 1733/2295] renamed ghcr_python_ubuntu.yaml to ghcr_python_ubuntu.yaml.disabled --- .../{ghcr_python_ubuntu.yaml => ghcr_python_ubuntu.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ghcr_python_ubuntu.yaml => ghcr_python_ubuntu.yaml.disabled} (100%) diff --git a/.github/workflows/ghcr_python_ubuntu.yaml b/.github/workflows/ghcr_python_ubuntu.yaml.disabled similarity index 100% rename from .github/workflows/ghcr_python_ubuntu.yaml rename to .github/workflows/ghcr_python_ubuntu.yaml.disabled From be9fcb18e46359a1befced1c8f96ef9d934ad53f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 1734/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a2dfe24e2..c07877011 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest From 002e00ed5fbd3d40ac861b88d7bfaf338735cf2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 1735/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 2a7fbaff0..e692d25b1 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 From 20a03a92addf21803a0c03bcf21f40b1e8bfd89e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 1736/2295] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index fbaf19c4f..d4147ea01 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest From 211ed21ca520d9754d2a4f018348db986f3141f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 1737/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index ebaf5fae2..5a588f28d 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 From 024e14c8e2c24e1645ac3d34de9afc9135f71a69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 1738/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index fada215de..56fdbba13 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 From ea3718ed1bc35ddf15dfe757ee6096bbdac46fd5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 1739/2295] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index e993c7222..98f6d7414 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest From dae14434eecd8f00ed847f645aa559dcd95f07da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 1740/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b120564e9..b45963ac3 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 From cb0fd02a788b269b924d816c226827c74a2ae4e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 1741/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 2e8c06e55..bcfb2efca 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 From 3b81f25ffc19f84aa4f4b902acca14d82f7ef399 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 1742/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 65fe55cca..f09fdf524 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 From c4930b32f7160b574b5133118fd479cd8c64f15a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1743/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index ae2bc9f9b..043bc389b 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 From ac421dffa9eabf26f492a3d84610ead2627507fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1744/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 9223474c1..6503faeeb 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 From 737e740ec8b3e50ad90eb9be15e57aff3944dddc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1745/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index fe6bcbe6f..71d54f26a 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora From 490e2a8ef76bbdab824ab66a2a69d293a13683c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1746/2295] updated mac.yaml --- .github/workflows/mac.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 92e326bcb..fef39007c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-latest From 542ec9e11554d900442ace3251d2e09f516e7ca0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1747/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index b8feb60c0..fcc669e97 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-10.15 From 48cb53b8f6da9cefd0d96c659f835a05254984f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 1748/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 707dd82f0..b6436c968 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -11,12 +11,24 @@ # https://www.linkedin.com/in/HariSekhon # +--- name: PyPy 2 on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -43,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: PyPy2 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: pypy2 + python-version: pypy2 debug: ${{ github.event.inputs.debug }} From 4d23cfcbc9d9d0b85b36aa2b1d89260824f0eca1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 1749/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index c379246c9..6c7d2a402 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -11,12 +11,24 @@ # https://www.linkedin.com/in/HariSekhon # +--- name: PyPy 3 on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -43,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: PyPy3 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: pypy3 + python-version: pypy3 debug: ${{ github.event.inputs.debug }} From 8de68c7f994ed783f09f88d23a10b6055dd9121c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 1750/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index a850b2b84..727e300f0 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 2.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 2.7 + python-version: 2.7 debug: ${{ github.event.inputs.debug }} From 92bb5125370cbf97d08ac5f237a9bd97f142df5d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 1751/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index da5b11f35..31bf6e30c 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.10 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: "3.10" + python-version: "3.10" debug: ${{ github.event.inputs.debug }} From 20ec0d435277be90e08b840163fcc8485a7b9a0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 1752/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 32e3089e2..3bb00c01d 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.6 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.6 + python-version: 3.6 debug: ${{ github.event.inputs.debug }} From 004bf873069e33a4748a44835a57d9a5ec404f99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 1753/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index d6efcf883..0d463f1f0 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.7 + python-version: 3.7 debug: ${{ github.event.inputs.debug }} From c920da17656d53446508dab674afdcae8a358ffc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1754/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f0eb8fa2b..4a1df9f81 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.8 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.8 + python-version: 3.8 debug: ${{ github.event.inputs.debug }} From c6236581ffe418815c6157e7a7884e079da991eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1755/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index df374c960..e4aa45470 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.9 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.9 + python-version: 3.9 debug: ${{ github.event.inputs.debug }} From a19439b6e56fd29c84334552ebdd1c30e7362c0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1756/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 6a623c6da..b7d2f2e1c 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest From 9a67e78b8e7e703ab60fdddb9794cff899391c30 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1757/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 915f04ae9..b38f1bfcd 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 From a4ee6d97c9f1c7366ffadea18e1ba8a5e6b6c1ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1758/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f459f4398..26802e404 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 From 46f1972f7732ec55e4d04216653bdf1ad9a8629a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 1759/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 926c7e0a5..70bc87eb6 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 From 50e84e49625b5481987bfa49f0dbaacc8247bcd7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:44 +0100 Subject: [PATCH 1760/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ad419846f..46fa80dc6 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 From 8a32413be768268fa1f469fc419199feb8448674 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:44 +0100 Subject: [PATCH 1761/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 85bb7dd48..5b736dffc 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: caches: apt pip cpanm From 22b6a697b212e0a4528d7c4b69556c1f8c121117 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 1762/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 07871b12f..9e7f34224 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 07871b12f83f5870d7ada82fa3de8b43ed0b56d4 +Subproject commit 9e7f342243b5514ed0d9b3e04228538d2ee98330 From c3a6cf2d9ce61ff83222f900aaf7722f2e9b8f37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 1763/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 965905dd6..d23609649 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 965905dd62b8c402f36a546c043fad07d4ae9d92 +Subproject commit d2360964912b643102ac73433cbbe8cc542d4c40 From 58ae9294804702179e0cff0d31c8a1c6cb969700 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 1764/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0c8255344..282fb3eb0 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0c825534472969e6a41770e9246ede7861dbbb10 +Subproject commit 282fb3eb06ee967f0bc308b4506b5d72d8cbccd0 From a886e32a52c31c0e863db35095d39f321c6fbb8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 1765/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e116df4dd..94b23ae15 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e116df4ddff7d917fa28291cbca1f1744c8bc664 +Subproject commit 94b23ae1575080d1771f67bf0f3ee4e0f490b99d From 83bedbaa955264ce3502b29ae53e07ac61596aca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 1766/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index b6436c968..d21626a61 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy2 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 420f6e3159ab1ce6a2cfe72a0bff6e0d4afadcfe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 1767/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 6c7d2a402..9c558f16a 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy3 + caches: apt pip debug: ${{ github.event.inputs.debug }} From faae953eb96cafa4af86ada0ecf4bc08608674bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 1768/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 727e300f0..707543308 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 2.7 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 3758fbb2015b2822587b5a8bc4570eea4781eb44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 1769/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 31bf6e30c..fda309782 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: "3.10" + caches: apt pip debug: ${{ github.event.inputs.debug }} From 69da3a6fb05a1d3fea8e3b99e70bc61cec19cd0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 1770/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 3bb00c01d..84c61bc5f 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.6 + caches: apt pip debug: ${{ github.event.inputs.debug }} From b68c897a33154a98c032f03e340caaaa6fda0246 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 1771/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 0d463f1f0..6a4191bcc 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.7 + caches: apt pip debug: ${{ github.event.inputs.debug }} From d099da0e894e4ef138049ef975be6a7279023059 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 1772/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 4a1df9f81..de4668018 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.8 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 337bb7202dfed79d5e35d4b8543ed55e31ff09b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 1773/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index e4aa45470..97914800c 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.9 + caches: apt pip debug: ${{ github.event.inputs.debug }} From d70b960e126ee5948f8e7b3425a0a2956368b6e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 1774/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9e7f34224..fc1c416f5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9e7f342243b5514ed0d9b3e04228538d2ee98330 +Subproject commit fc1c416f5a808fc5bf42b369c46e70ee3410286e From 937b9981b9663a090a6640057b089a0cf22c7b60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 1775/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d23609649..3bf8af87c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d2360964912b643102ac73433cbbe8cc542d4c40 +Subproject commit 3bf8af87c27ed8e9babef1bbd5c414aa0678e021 From 43fd53a7c4dd592c7e2d10b3acc839e72e3ecc4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 1776/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 94b23ae15..26c0a409b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 94b23ae1575080d1771f67bf0f3ee4e0f490b99d +Subproject commit 26c0a409b8d18d3fdea3eb68db5d8c8af87908f7 From 04c3bb05d4bb816769795536bd137752efc36dcc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 18:41:22 +0100 Subject: [PATCH 1777/2295] updated README.md --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b972256b0..3aa702615 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,11 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) -[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) -[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) -[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) +[![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) @@ -24,6 +15,17 @@ Hari Sekhon - DevOps Python Tools [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) --> +[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) +[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) + [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) [![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/github/) From e332dd04dd88f32c49b75e9b8962c4537472c929 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:23:28 +0100 Subject: [PATCH 1778/2295] added .werckerignore --- .werckerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore new file mode 100644 index 000000000..ff8fb8247 --- /dev/null +++ b/.werckerignore @@ -0,0 +1 @@ +**/.md From 73c1f49ab95c412844cafa43cd4971e23cd227a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:24:53 +0100 Subject: [PATCH 1779/2295] updated .cirrus.yml --- .cirrus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cirrus.yml b/.cirrus.yml index 46de8ac49..7ea506862 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,6 +19,7 @@ container: image: ubuntu:18.04 task: + skip: "!changesInclude('**/*.md')" script: - setup/ci_bootstrap.sh - make init From 6675defaaecc6592ebe734de7f6ee91e81a86464 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:25:30 +0100 Subject: [PATCH 1780/2295] updated .appveyor.yml --- .appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f23536e32..1551a3473 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,6 +17,11 @@ image: Ubuntu +skip_commits: + files: + - docs/* + - '**/*.md' + # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ environment: APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension From a5b27a7d95d37062038dabd5c300afd6b39baee3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:26:08 +0100 Subject: [PATCH 1781/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6e5dc9f25..99040a203 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -26,7 +26,7 @@ execution_time_limit: blocks: - name: Linux build run: - when: "branch = 'master'" + when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})" #execution_time_limit: # hours: 2 task: From 33613dcb26325f085651eb7aa1f84517c65aac32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:35:27 +0100 Subject: [PATCH 1782/2295] updated .cirrus.yml --- .cirrus.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7ea506862..7f1425a4c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,7 +19,8 @@ container: image: ubuntu:18.04 task: - skip: "!changesInclude('**/*.md')" + # doesn't work properly + #skip: "!changesInclude('**/*.md')" script: - setup/ci_bootstrap.sh - make init From 54a6e3b9b8a94ee0161a84ebb3fdd7aafc9f0855 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 1783/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fc1c416f5..9d6404984 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fc1c416f5a808fc5bf42b369c46e70ee3410286e +Subproject commit 9d640498465c747e94e8e2e281e252af2cf3096f From cdaef5573a767373b4e49dd36e583c83c462b4e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 1784/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3bf8af87c..c7887c4bb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3bf8af87c27ed8e9babef1bbd5c414aa0678e021 +Subproject commit c7887c4bbce54d0cbc2dec40676fcf39209ad992 From fbcbb926aea17b127c413706e041e05c6a312473 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 1785/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 282fb3eb0..fb98e0a2f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 282fb3eb06ee967f0bc308b4506b5d72d8cbccd0 +Subproject commit fb98e0a2fa84f94916d02336d9eee323e96e34b9 From 3fcc6510f18532fbf15dffd8d20cbb299728f638 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 1786/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 26c0a409b..9a8db2f6e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 26c0a409b8d18d3fdea3eb68db5d8c8af87908f7 +Subproject commit 9a8db2f6e480766c35ad3332171cfa0aaa697d1a From 615319007021fd79410ffb215bcb65a7c62baefb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 11 Apr 2022 15:58:48 +0100 Subject: [PATCH 1787/2295] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 66b95292f..b7f961b67 100644 --- a/Makefile +++ b/Makefile @@ -210,7 +210,8 @@ test-lib: cd pylib && $(MAKE) test .PHONY: test -test: test-lib +#test: test-lib +test: tests/all.sh .PHONY: basic-test From 1a9f9b484f6485ac508868005e9e7239f49ad825 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:39:23 +0100 Subject: [PATCH 1788/2295] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 556bd80d9..cad44a536 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 9 * * *' + - cron: '30 9 * * 2' permissions: contents: write From 1099646495a4a0779ac722f18fd52e2ac4acde67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:39:24 +0100 Subject: [PATCH 1789/2295] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 12449fb36..1897533a2 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,8 +22,8 @@ on: required: false default: false schedule: - # fork-sync happens for default branch every hour, so just after that, run PRs for branches - - cron: '2 10 * * 2' + # fork-sync happens 9:30 every tuesday, so after that run PRs for branches + - cron: '0 10 * * 2' permissions: contents: write From b6161858a07af7d574f5bf8c3c32eaa70cadd34e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 1790/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9d6404984..62a152225 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9d640498465c747e94e8e2e281e252af2cf3096f +Subproject commit 62a152225697e0c05244aa52ab875dc2978baf20 From 6316b262c9cbda7f73db4a57112eaf5adce021ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 1791/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c7887c4bb..8ae4d15e2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c7887c4bbce54d0cbc2dec40676fcf39209ad992 +Subproject commit 8ae4d15e20b0f061385485abea2369e2b3e634e6 From 6cbd8eaea51bd48bb4865c9f3e68a8dab207dc82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 1792/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fb98e0a2f..6f5a26d58 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fb98e0a2fa84f94916d02336d9eee323e96e34b9 +Subproject commit 6f5a26d58504258b2749a8316ae92b3836d27bf8 From 6874f264b882f5218d5002d8e67ee09aa8829f10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 1793/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9a8db2f6e..68441f7ed 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9a8db2f6e480766c35ad3332171cfa0aaa697d1a +Subproject commit 68441f7edeeedd5408c95e3969eaa0f3721b5296 From afaabdab1a225543a8a1ed5d183debc3e456b312 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 1794/2295] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index c07877011..39e4a4497 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 83d72dd0ebc14e8a3c59d6d99af36f33b09d6e1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 1795/2295] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index e692d25b1..0b1cbea05 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4ed362a170e37faa5313e28b30019e9e623cb576 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 1796/2295] updated centos.yaml --- .github/workflows/centos.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index d4147ea01..6f3dd2b27 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From f820794229afbb69248abbef61b1857162052f16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 1797/2295] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5a588f28d..46c3ca212 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From a28ea1564a505b0ae01555f5e85215215ba02170 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 1798/2295] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 56fdbba13..782ab12ff 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 13c981adaa13d0fd56cd93a313da5e68c96e86b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1799/2295] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 26be616d6..8f31b2672 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,7 +51,9 @@ permissions: jobs: checkov: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: From 0122b4e055177136184ebafc95fccf2929eafbcf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1800/2295] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 934d09b92..5fdb3401e 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -47,6 +47,8 @@ concurrency: jobs: validate: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master From a44c53e059acecf61c0f4534fb5b9c8b32b1847e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1801/2295] updated debian.yaml --- .github/workflows/debian.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 98f6d7414..6ded416ad 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 47a6dd8652512e6409d43862f934431f8fd70993 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1802/2295] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b45963ac3..debc3945d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 913f5c50420810c7289add8da7610690e6eefad0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1803/2295] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index bcfb2efca..1cb4e5431 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 6eb8c5a5a2188ad2b331c70fe0344a7cbacd3702 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1804/2295] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index f09fdf524..e987d7e46 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 45bc93d322d763fb16fa99cf07686ee70a6e98e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1805/2295] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 043bc389b..e8fc5595a 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 04c216e6037e3558ff2ae3be261d627b7d1d349f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1806/2295] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6503faeeb..5c14364d9 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 000a39bda485bf53f61c4f92573caff780115946 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 1807/2295] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 71d54f26a..0c29aeccb 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 615ae322dbb8ec544ef53e584c0eab2e2fb5aa2a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 1808/2295] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index cad44a536..f1e605236 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -33,6 +33,8 @@ concurrency: jobs: fork_sync: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master From 61cb4a4f7a7f81ea7c16769d6fe84f512be378f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 1809/2295] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 1897533a2..1767b3e33 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -35,6 +35,8 @@ concurrency: jobs: fork_update_pr: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master From d03abe30fdf50b23c05b8276d9cc3f25df81c1e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 1810/2295] updated json.yaml --- .github/workflows/json.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 5a9df06b5..5d70a6e2a 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -41,6 +41,8 @@ permissions: jobs: check_json: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master From b42654902236e939461b57f76655bc185a061589 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 1811/2295] updated mac.yaml --- .github/workflows/mac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index fef39007c..06e696b85 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 367958487167f57e5a1070c7de6a597e218c66c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1812/2295] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index fcc669e97..4f42839e7 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 3d33c7d16aea8d3d7e6780959b26b1a6187d0d95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1813/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index d21626a61..ff7ebfc1f 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 0a66d7b7646a10df5a2e9c852e7015e33ca2e777 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1814/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 9c558f16a..be648b6b7 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From a93c9abd53b99757035206d12eaf6bfec5f84cdf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1815/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 707543308..6f6a3cc13 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 352a8ad47386e38b2771bb75df4d0f23a39c0880 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1816/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index fda309782..c7710cc6d 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 38b2abd4f786a6623b4272cc6bbaf9cb6b63ee29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1817/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 84c61bc5f..a37f4b538 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4605089e36a3d17be5b4e9d805fcec38326296e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1818/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 6a4191bcc..8c19509a2 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4cd40000afa4f288642c08881c98b3a7a8417c70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1819/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index de4668018..025424851 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 1e8f850467fc1e016ae78214be928ed74916b7ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1820/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 97914800c..862e2eccf 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.9 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From e1599fcee8a4ea5b9994bfe2db2216038767ec3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1821/2295] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 47fe5fcf5..9bb52df1a 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -51,7 +51,9 @@ concurrency: jobs: semgrep: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: From 26983c4b3328b23e0f7dece4e4e4399ef87c831b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1822/2295] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index ea33c7f40..cacc83e09 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -55,6 +55,8 @@ concurrency: jobs: semgrep: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master From db5dc7a745c81da469a2a21831c4909095db5a09 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 1823/2295] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index b7d2f2e1c..d6acb8902 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 81a4ff08c879b0f243c9a77a95164bae199c46bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1824/2295] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index b38f1bfcd..da84dee9c 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 2745c6081b0003bc43c800c77d337c3e278df4aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1825/2295] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 26802e404..a77170ded 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 08c56418776425a67ae79de8ff474c7fd9748e06 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1826/2295] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 70bc87eb6..5e66ef8d3 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 19f3b31a1875a5cc11b7d69a9e6307998306adec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1827/2295] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 46fa80dc6..7da14d8bc 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From acfddac39d122371223af906eeb7f5fdbaa489c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1828/2295] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 5b736dffc..5a24bd177 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 21adc74146717bbeca517c5e4856ec749a201b2a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1829/2295] updated validate.yaml --- .github/workflows/validate.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index b9634e4ae..a2daad965 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -45,6 +45,8 @@ concurrency: jobs: validate: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From 7767f1c7d3a77a1094c1d5d476b9fbca8eca70c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 1830/2295] updated yaml.yaml --- .github/workflows/yaml.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 42f1aae0e..81a6578a6 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -47,6 +47,8 @@ concurrency: jobs: check_yaml: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master From 6b71f80bd0b4c07e0a54f46a3219e993ac2fe9fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 19:32:08 +0100 Subject: [PATCH 1831/2295] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index f1e605236..4d4ebd577 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '30 9 * * 2' + - cron: '0 */3 * * *' permissions: contents: write From 00a4cadd8711a2ad22e4e43f89cc3eddb1ff9335 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 19:32:08 +0100 Subject: [PATCH 1832/2295] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 1767b3e33..eff67422c 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,6 @@ on: required: false default: false schedule: - # fork-sync happens 9:30 every tuesday, so after that run PRs for branches - cron: '0 10 * * 2' permissions: From 899f6498e290a1c65f8ec31e33a0a6ea34a4a712 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 1833/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 62a152225..c9878cdf5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 62a152225697e0c05244aa52ab875dc2978baf20 +Subproject commit c9878cdf5cb52eea7366c73967e415a2a25cb62a From d7678a89275e50e4022eff301e27cbc20dd3c1a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 1834/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8ae4d15e2..1dc4ec7c5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8ae4d15e20b0f061385485abea2369e2b3e634e6 +Subproject commit 1dc4ec7c57ea82317910cb6d9a550837af120a68 From c749b62f069144e277b62a12e0a0e4a3ed98b2d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 1835/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 6f5a26d58..9ac92d3dc 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 6f5a26d58504258b2749a8316ae92b3836d27bf8 +Subproject commit 9ac92d3dcb3c2f72d0b9ab896604e932f5d43aa7 From 17c617a928901140f5262de94b77ee1b5188d815 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 1836/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 68441f7ed..7330e7ab9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 68441f7edeeedd5408c95e3969eaa0f3721b5296 +Subproject commit 7330e7ab958a08a3f22a70dfdb4d807dc9276d74 From 14923853f2771751c179cd867a0d94b9ea418bac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 18:44:33 +0100 Subject: [PATCH 1837/2295] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index eff67422c..00479ecb7 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 10 * * 2' + - cron: '0 10 * * 1' permissions: contents: write From af2e0f141749df2aeac0f488ebbd129453d5ecdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 1838/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c9878cdf5..4bc47ad4e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c9878cdf5cb52eea7366c73967e415a2a25cb62a +Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 From 1999b1bbabccf27c89a7ea9c06dfaff42bce782d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 1839/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1dc4ec7c5..b22457e26 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1dc4ec7c57ea82317910cb6d9a550837af120a68 +Subproject commit b22457e26204158f537524111be4e45669b82663 From 504eb8b6c3a39f38e2b683a9328e988dd4da7a77 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 1840/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 9ac92d3dc..300fd2a80 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 9ac92d3dcb3c2f72d0b9ab896604e932f5d43aa7 +Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 From 749bd6a88cd56ff6c69c59d0aa0e44e282f6b8ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 1841/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7330e7ab9..9a3f5c616 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7330e7ab958a08a3f22a70dfdb4d807dc9276d74 +Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f From ed6feacd0214d212eaa0eb96a55fecde0798809b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 1842/2295] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index ff7ebfc1f..7dbc25c7f 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 3e3a05b93f242a81ae2943d73087e9ff2085e529 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 1843/2295] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index be648b6b7..ac669813c 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 058b98407d0b1b131ba6e9ce01ea2059c5a6ef0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 1844/2295] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 6f6a3cc13..60b31b6bc 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 973c93960cd0793efa8f9dd739c4c5249ae8ea66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 1845/2295] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index c7710cc6d..939ae7337 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6516c90d6b7eb00bc64f96b00fb14acd049f0b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 1846/2295] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index a37f4b538..d6a61ea45 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6bcf6787f950225bf13bce535cd23accce2a1f62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 1847/2295] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 8c19509a2..70ca1f828 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From edb68351bd5881970da4e00050cd28a202fb53a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 1848/2295] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 025424851..fd6dcefb2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 11bf3fe84c23f316d212bd93bcde98cbd237bc4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:29 +0100 Subject: [PATCH 1849/2295] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 862e2eccf..e8d331278 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 9180bb752833e6178946378de108889c49fcd9dc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:37:12 +0100 Subject: [PATCH 1850/2295] added pylib and requirements.txt --- .github/workflows/python2.7.yaml | 6 ++++++ .github/workflows/python3.10.yaml | 6 ++++++ .github/workflows/python3.6.yaml | 6 ++++++ .github/workflows/python3.7.yaml | 6 ++++++ .github/workflows/python3.8.yaml | 6 ++++++ .github/workflows/python3.9.yaml | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 60b31b6bc..ff3e51dd1 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 939ae7337..de6f79440 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d6a61ea45..84dce85e0 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 70ca1f828..58ad6c210 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index fd6dcefb2..f1c806dc2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index e8d331278..31c7d0ab8 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml workflow_dispatch: inputs: debug: From 2fe17870b260fb815a2db3e63c0e7138b925f6b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 1851/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4bc47ad4e..b352785d0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 +Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb From 1f7dd4b40204da6a26f91bb930fbf3fb36674694 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 1852/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b22457e26..ffe46e95e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b22457e26204158f537524111be4e45669b82663 +Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b From 1d29f187221e74e7e05c9322a00402b8f9d865a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 1853/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 300fd2a80..ad25bcb23 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 +Subproject commit ad25bcb23280f7307d8897111a238cc6b688f876 From 2ae16a579a126f1d43d69d9f4748a83bef4ec77a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 1854/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9a3f5c616..de060e1aa 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f +Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb From 13675e4f89ec8b98b31c552cca802c7e3199c4c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 1855/2295] updated docker_pytools_alpine.yaml --- .github/workflows/docker_pytools_alpine.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index c3687b351..252e76f7b 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-alpine + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 3204f55973046b07e10b5148d85074e6a0498dbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 1856/2295] updated docker_pytools_centos.yaml --- .github/workflows/docker_pytools_centos.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 92b36e7d5..88474229f 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-centos + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 4fb936a61ebc471c3abb1bc378235ce3c343eeaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 1857/2295] updated docker_pytools_debian.yaml --- .github/workflows/docker_pytools_debian.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 8b34df315..876ccce1f 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-debian + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From d4196934562d74e6a72599ec297fccd8003c1d10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 1858/2295] updated docker_pytools_fedora.yaml --- .github/workflows/docker_pytools_fedora.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 190cf703b..0299270f3 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-fedora + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 9e62e14fb9aa565b9dc08bb46e8f8814df9d4bfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:32 +0100 Subject: [PATCH 1859/2295] updated docker_pytools_ubuntu.yaml --- .github/workflows/docker_pytools_ubuntu.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index fe5e51a95..535621a81 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -33,6 +48,7 @@ jobs: ghcr.io/harisekhon/pytools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-ubuntu + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 898775ec901b4d4cb8ad6204c98c59a69fdd8d1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:01:47 +0100 Subject: [PATCH 1860/2295] added kics.yaml --- .github/workflows/kics.yaml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/kics.yaml diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 000000000..b73bedf6f --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Kics + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master From 6216db9880e0ce935a1e264566dad52a911a1e3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:05:30 +0100 Subject: [PATCH 1861/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3aa702615..1561bd333 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Hari Sekhon - DevOps Python Tools [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) From f3cc7f678f6a75a4dba04b47ac38c59a5ae28d9a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 1862/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b352785d0..df5a509bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb +Subproject commit df5a509bb8c679914a137c8ff21597863fdd21ac From 131c4a984cb3848411c15b2036216c70ec5dbfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 1863/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ffe46e95e..548ebc658 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b +Subproject commit 548ebc658c20f56600cac1bbd87ee1f267274696 From fbed6887015c0207021ef28ce82a4376260a009d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 1864/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index de060e1aa..b81a01bca 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb +Subproject commit b81a01bca4cca15dd9cf495bb253d1e27693cc7d From d3b9407383090a554a75ec40f452155c4895765b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 May 2022 12:01:24 +0100 Subject: [PATCH 1865/2295] updated URLs --- .appveyor.yml | 4 +- .buildkite/pipeline.yml | 4 +- .circleci/config.yml | 4 +- .cirrus.yml | 4 +- .concourse.yml | 6 +-- .drone.yml | 4 +- .editorconfig | 2 +- .github/ISSUE_TEMPLATE.md | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_centos.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .../ghcr_python_ubuntu.yaml.disabled | 2 +- .gitlab-ci.yml | 2 +- .gitmodules | 2 +- .gocd.yml | 6 +-- .semaphore/semaphore.yml | 4 +- .travis.yml | 4 +- .yamllint | 4 +- Jenkinsfile | 6 +-- Makefile | 10 ++--- README.md | 38 +++++++++---------- ambari_ams_metrics.sh | 4 +- ambari_blueprints.py | 4 +- ambari_cancel_all_requests.sh | 4 +- ambari_trigger_service_checks.py | 4 +- anonymize.py | 8 ++-- anonymize_custom.conf | 8 +++- anonymize_ignore.conf | 2 +- anonymize_parallel.sh | 4 +- aws_s3_presign.py | 4 +- aws_users_access_key_age.py | 8 ++-- aws_users_last_used.py | 8 ++-- aws_users_pw_last_used.py | 8 ++-- aws_users_unused_access_keys.py | 8 ++-- azure-pipelines.yml | 2 +- bitbucket-pipelines.yml | 4 +- buddy.yml | 6 +-- buildspec.yml | 2 +- center.py | 4 +- cloudbuild.yaml | 2 +- cloudera_navigator_tables_used.py | 4 +- cloudera_navigator_tables_used_postgres.py | 4 +- codefresh.yml | 4 +- codeship.yml | 2 +- crunch_accounting_csv_statement_converter.py | 4 +- ...unch_accounting_csv_statement_converter.sh | 2 +- ...unch_accounting_csv_statement_converter.sh | 2 +- .../lib.sh | 2 +- docker_pull_all_images.sh | 4 +- docker_pull_all_images_all_tags.sh | 4 +- docker_pull_all_tags.sh | 4 +- docker_registry_show_tags.py | 4 +- dockerfiles_check_git_branches.py | 4 +- dockerfiles_check_git_tags.py | 4 +- dockerhub_search.py | 6 +-- dockerhub_show_tags.py | 4 +- find_active_apache_drill.py | 4 +- find_active_apache_drill2.py | 4 +- find_active_cassandra.py | 4 +- find_active_elasticsearch.py | 4 +- find_active_hadoop2_namenode.py | 4 +- find_active_hadoop_namenode.py | 4 +- find_active_hadoop_yarn_resource_manager.py | 4 +- find_active_hbase_master.py | 4 +- find_active_hbase_stargate.py | 4 +- find_active_hbase_thrift.py | 4 +- find_active_impala.py | 4 +- find_active_impala_catalog.py | 4 +- find_active_impala_statestore.py | 4 +- find_active_kubernetes_api.py | 4 +- find_active_oozie.py | 4 +- find_active_presto_coordinator.py | 4 +- find_active_server.py | 6 +-- find_active_solrcloud.py | 4 +- find_duplicate_files.py | 4 +- find_missing_files_in_sequence.py | 2 +- getent.py | 4 +- git_check_branches_upstream.py | 6 +-- hbase_compact_tables.py | 4 +- hbase_flush_tables.py | 4 +- hbase_generate_data.py | 4 +- hbase_region_requests.py | 4 +- hbase_regions_by_memstore_size.py | 4 +- hbase_regions_by_size.py | 4 +- hbase_regions_least_used.py | 4 +- hbase_regionserver_requests.py | 4 +- hbase_scan_table_column_names.sh | 4 +- hbase_show_table_region_ranges.py | 4 +- hbase_table_region_row_distribution.py | 4 +- hbase_table_regions_by_regionserver.sh | 4 +- hbase_table_regionserver_distribution.sh | 4 +- hbase_table_row_key_distribution.py | 4 +- hbase_uncompacted_regions.sh | 4 +- hdfs_files_native_checksums.jy | 4 +- hdfs_files_stats.jy | 4 +- hdfs_find_replication_factor_1.py | 4 +- hdfs_time_block_reads.jy | 4 +- headtail.py | 4 +- hexanonymize.py | 2 +- hive_compute_table_stats.py | 4 +- hive_foreach_table.py | 4 +- hive_schemas_csv.py | 4 +- hive_tables_column_counts.py | 4 +- hive_tables_list.py | 4 +- hive_tables_locations.py | 4 +- hive_tables_metadata.py | 4 +- hive_tables_null_columns.py | 4 +- hive_tables_null_rows.py | 4 +- hive_tables_row_column_counts.py | 4 +- hive_tables_row_counts.py | 4 +- hive_tables_row_counts_any_nulls.py | 4 +- impala_compute_table_stats.py | 4 +- impala_foreach_table.py | 4 +- impala_schemas_csv.py | 4 +- impala_tables_column_counts.py | 4 +- impala_tables_list.py | 4 +- impala_tables_locations.py | 4 +- impala_tables_metadata.py | 4 +- impala_tables_null_columns.py | 4 +- impala_tables_null_rows.py | 4 +- impala_tables_row_column_counts.py | 4 +- impala_tables_row_counts.py | 4 +- impala_tables_row_counts_any_nulls.py | 4 +- ipython_notebook_pyspark.py | 4 +- json_docs_to_bulk_multiline.py | 4 +- json_to_xml.py | 2 +- json_to_yaml.py | 2 +- jython_autoinstall.exp | 4 +- jython_install.sh | 4 +- lib/hive_impala_cli.py | 4 +- lib/postgres_cli.py | 4 +- opentsdb_import_metric_distribution.py | 4 +- opentsdb_list_metrics.sh | 4 +- opentsdb_list_metrics_hbase.sh | 4 +- pig_udfs.jy | 2 +- python_find_library_path.py | 4 +- pythonpath.py | 4 +- quay_show_tags.py | 4 +- serf_event_handler.py | 4 +- setup/apk-packages-dev.txt | 4 +- setup/apk-packages-pip.txt | 4 +- setup/apk-packages.txt | 4 +- setup/bootstrap.sh | 6 +-- setup/brew-packages.txt | 4 +- setup/ci_bootstrap.sh | 4 +- setup/deb-packages-dev.txt | 4 +- setup/deb-packages-optional.txt | 4 +- setup/deb-packages-pip.txt | 4 +- setup/deb-packages.txt | 4 +- setup/gocd_config_repo.json | 2 +- setup/jenkins-job.xml | 4 +- setup/rpm-packages-dev.txt | 4 +- setup/rpm-packages-optional.txt | 4 +- setup/rpm-packages-pip.txt | 4 +- setup/rpm-packages.txt | 4 +- shippable.yml | 4 +- sonar-project.properties | 10 ++--- spark_avro_to_parquet.py | 4 +- spark_csv_to_avro.py | 4 +- spark_csv_to_parquet.py | 4 +- spark_json_to_avro.py | 4 +- spark_json_to_parquet.py | 4 +- spark_parquet_to_avro.py | 4 +- strip_ansi_escape_codes.py | 4 +- tests/all.sh | 4 +- tests/check.sh | 4 +- tests/compile.sh | 4 +- tests/data/add_ou.ldif | 4 +- tests/data/ldap_download.sh | 2 +- tests/data/ldap_upload.sh | 2 +- tests/docker/apache-drill-docker-compose.yml | 4 +- tests/docker/common.yml | 4 +- tests/docker/elasticsearch-docker-compose.yml | 4 +- ...lasticsearch-elastic.co-docker-compose.yml | 4 +- tests/docker/hadoop-docker-compose.yml | 4 +- tests/docker/hbase-docker-compose.yml | 4 +- tests/docker/presto-dev-docker-compose.yml | 4 +- tests/docker/presto-docker-compose.yml | 4 +- tests/docker/registry-docker-compose.yml | 4 +- tests/docker/solrcloud-docker-compose.yml | 4 +- tests/excluded.sh | 4 +- tests/help.sh | 4 +- tests/python3.sh | 4 +- tests/syntax.sh | 4 +- tests/test_anonymize.sh | 4 +- tests/test_apache-drill.sh | 4 +- tests/test_center.sh | 2 +- tests/test_docker.sh | 4 +- tests/test_docker_registry_show_tags.sh | 4 +- tests/test_dockerfiles_check_git_branches.sh | 6 +-- tests/test_dockerfiles_check_git_tags.sh | 6 +-- tests/test_dockerhub_search.sh | 4 +- tests/test_dockerhub_show_tags.sh | 4 +- tests/test_elasticsearch.sh | 4 +- tests/test_find_active_server.sh | 2 +- tests/test_find_duplicate_files.sh | 4 +- tests/test_find_python_library_path.sh | 4 +- tests/test_getent.sh | 4 +- tests/test_git_check_branches_upstream.sh | 6 +-- tests/test_hadoop.sh | 4 +- tests/test_hbase.sh | 4 +- tests/test_headtail.sh | 4 +- tests/test_json.sh | 4 +- tests/test_json_docs_to_bulk_multiline.sh | 4 +- tests/test_json_to_xml.sh | 4 +- tests/test_json_to_yaml.sh | 4 +- tests/test_opentsdb.sh | 4 +- tests/test_presto.sh | 4 +- tests/test_quay_show_tags.sh | 4 +- tests/test_serf_event_handler.sh | 4 +- tests/test_solrcloud.sh | 4 +- tests/test_spark_csv_to_avro.sh | 4 +- tests/test_spark_csv_to_parquet.sh | 4 +- tests/test_spark_json_to_avro.sh | 4 +- tests/test_spark_json_to_parquet.sh | 4 +- tests/test_spark_z_avro_to_parquet.sh | 4 +- tests/test_spark_z_parquet_to_avro.sh | 4 +- tests/test_timeout.sh | 2 +- tests/test_validate_avro.sh | 4 +- tests/test_validate_cson.sh | 4 +- tests/test_validate_csv.sh | 4 +- tests/test_validate_ini.sh | 4 +- tests/test_validate_json.sh | 4 +- tests/test_validate_ldap_ldif.sh | 4 +- tests/test_validate_multimedia.sh | 4 +- tests/test_validate_parquet.sh | 4 +- tests/test_validate_toml.sh | 4 +- tests/test_validate_xml.sh | 4 +- tests/test_validate_yaml.sh | 4 +- tests/test_welcome.sh | 4 +- tests/test_xml_to_json.sh | 4 +- tests/test_xml_to_yaml.sh | 4 +- tests/test_yamllint.sh | 4 +- tests/utils.sh | 4 +- timeout.py | 4 +- travis_debug_session.py | 4 +- travis_last_log.py | 4 +- urldecode.py | 4 +- urlencode.py | 4 +- validate_avro.py | 4 +- validate_cson.py | 4 +- validate_csv.py | 4 +- validate_ini.py | 4 +- validate_ini2.py | 4 +- validate_json.py | 4 +- validate_ldap_ldif.py | 4 +- validate_multimedia.py | 4 +- validate_parquet.py | 4 +- validate_toml.py | 4 +- validate_xml.py | 4 +- validate_yaml.py | 4 +- welcome.py | 4 +- wercker.yml | 4 +- xml_to_json.py | 2 +- xml_to_yaml.py | 2 +- yaml_to_json.py | 2 +- 258 files changed, 532 insertions(+), 528 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1551a3473..b706d6f72 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://www.appveyor.com/docs/appveyor-yml/ diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 53dd7405b..93f047c02 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # BuildKite Pipeline diff --git a/.circleci/config.yml b/.circleci/config.yml index 37332c73c..5dbde36b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,13 +5,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Master Template with more advanced config: diff --git a/.cirrus.yml b/.cirrus.yml index 7f1425a4c..d544aeb85 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://cirrus-ci.org/guide/writing-tasks/ diff --git a/.concourse.yml b/.concourse.yml index 9ed7da3e2..42b806f26 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # resources: @@ -18,7 +18,7 @@ resources: icon: github-circle type: git source: - uri: https://github.com/harisekhon/devops-python-tools + uri: https://github.com/HariSekhon/DevOps-Python-tools branch: master #- name: daily # type: time diff --git a/.drone.yml b/.drone.yml index 790bdcf4c..4d36c9db2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,13 +6,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.drone.io/quickstart/cli/ diff --git a/.editorconfig b/.editorconfig index 268166e72..aac36b310 100644 --- a/.editorconfig +++ b/.editorconfig @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5f04c0a58..aa46b9847 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. -You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. +You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 252e76f7b..8ce10e111 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 88474229f..e8db3ff9c 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 876ccce1f..2b92c8282 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 0299270f3..92a93a6b4 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index 535621a81..d6a4ddcb0 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled index fdd5ad61b..c4503bc13 100644 --- a/.github/workflows/ghcr_python_ubuntu.yaml.disabled +++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af2a6a22b..f9a691721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: Sun Feb 23 19:02:10 2020 +0000 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.gitmodules b/.gitmodules index 2bb07ff73..455bce3c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "pylib"] path = pylib - url = https://github.com/harisekhon/pylib + url = https://github.com/HariSekhon/pylib branch = master [submodule "bash-tools"] path = bash-tools diff --git a/.gocd.yml b/.gocd.yml index 69d289bf4..d3a89a0ef 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://github.com/tomzo/gocd-yaml-config-plugin#setup @@ -26,7 +26,7 @@ pipelines: display_order: -1 materials: git: - git: https://github.com/harisekhon/devops-python-tools + git: https://github.com/HariSekhon/DevOps-Python-tools shallow_clone: false auto_update: true branch: master diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 99040a203..dd256f26c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ diff --git a/.travis.yml b/.travis.yml index 71348b0bc..47f2f80b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,14 @@ # Author: Hari Sekhon # Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.yamllint b/.yamllint index b5e6f8448..e29153b7f 100644 --- a/.yamllint +++ b/.yamllint @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/Jenkinsfile b/Jenkinsfile index 7739f92e3..c4ff34fb2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,13 +4,13 @@ // Author: Hari Sekhon // Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) // -// https://github.com/harisekhon/devops-python-tools +// https://github.com/HariSekhon/DevOps-Python-tools // // License: see accompanying Hari Sekhon LICENSE file // // If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish // -// https://www.linkedin.com/in/harisekhon +// https://www.linkedin.com/in/HariSekhon // // ========================================================================== // @@ -49,7 +49,7 @@ pipeline { stages { stage ('Checkout') { steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]]) } } diff --git a/Makefile b/Makefile index b7f961b67..ae61cbc5e 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs. @@ -21,15 +21,15 @@ # # Alpine: # -# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # Debian / Ubuntu: # -# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # RHEL / CentOS: # -# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # =================== diff --git a/README.md b/README.md index 1561bd333..535e5389a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ Hari Sekhon - DevOps Python Tools ================================= -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) @@ -116,10 +116,10 @@ DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help` See Also: - - [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - - [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools) + - [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) + - [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) - [DevOps Golang Tools](https://github.com/HariSekhon/DevOps-Golang-tools) - - [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. @@ -127,7 +127,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### @@ -156,7 +156,7 @@ curl -L https://git.io/python-bootstrap | sh or manually: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools make ``` @@ -166,9 +166,9 @@ To only install pip dependencies for a single script, you can just type make and make anonymize.pyc ``` -Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. +Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information. -Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. +Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details. ### Usage @@ -290,7 +290,7 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI @@ -321,7 +321,7 @@ The automated build will use 'sudo' to install required Python PyPI libraries to Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools git submodule init git submodule update @@ -333,7 +333,7 @@ sudo pip install -r requirements.txt Download the DevOps Python Tools and Pylib git repos as zip files: -https://github.com/HariSekhon/devops-python-tools/archive/master.zip +https://github.com/HariSekhon/DevOps-Python-tools/archive/master.zip https://github.com/HariSekhon/pylib/archive/master.zip @@ -433,7 +433,7 @@ Then add the Jython install bin directory to the $PATH or specify the full path #### Configuration for Strict Domain / FQDN validation #### -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). #### Python SSL certificate verification problems @@ -457,9 +457,9 @@ If you update often and want to just quickly git pull + submodule update but ski ### Testing [Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios: -- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib) +- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib) - integration tests of the top level programs using the libraries for things like option parsing -- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) +- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) To trigger all tests run: @@ -475,7 +475,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery @@ -483,7 +483,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments -- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) - [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... @@ -491,9 +491,9 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 -- [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +- [PyLib](https://github.com/HariSekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule -- [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +- [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library From bccc01b7473cf51d3a5be02a09f379fe3ff569fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 2003/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0caddf7a5..5e7dfbdc8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0caddf7a58297dd75d19ecfea71eb2206853d4c4 +Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c From 5a0a33122adaa05720ea69222186feae21887c6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 2004/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 357fcd97c..4515bb958 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 357fcd97c3ccf3b6f89aa07ef71c994504e7c7c8 +Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b From cf3544f48c0168b9be909b530a3c071d9b46aad7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 2005/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ff707a116..d4eedc0e2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ff707a1168889fb3ad759c030f3fc633d23d800f +Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 From 194ca3daa90416a88ee399a7b9e7c3f0d7bd90c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 2006/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7373f570f..8d838daa6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7373f570ffb1a790ec576ef4c6cf52dcf0785ba0 +Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf From 2b13a79338d74de5b5ea78f59cd02f79484e1c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:28:26 +0100 Subject: [PATCH 2007/2295] renamed debian_8.yaml to debian_8.yaml.disabled --- .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} (100%) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml.disabled similarity index 100% rename from .github/workflows/debian_8.yaml rename to .github/workflows/debian_8.yaml.disabled From 8ef52b84f1c5e3aa90439b660ec903ee8c132e05 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:37:40 +0100 Subject: [PATCH 2008/2295] added debian_11.yaml --- .github/workflows/debian_11.yaml | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/debian_11.yaml diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..de3b4b96f --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From ed38e3efe36f2d5fc3eae3f90b6f50d82d538a47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:49:50 +0100 Subject: [PATCH 2009/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 152556544..af9216972 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) -[![Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) [![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) [![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) [![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) From 5ba4480dc89627d3c39d1f8de03f3450d709fa7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:42:16 +0100 Subject: [PATCH 2010/2295] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index af9216972..c6ae3495f 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,9 @@ [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From 824e98dd0d38a3382c69dac5eb7e85b1056836ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:30 +0100 Subject: [PATCH 2011/2295] renamed python3.6.yaml to python3.6.yaml.disabled --- .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} (100%) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml.disabled similarity index 100% rename from .github/workflows/python3.6.yaml rename to .github/workflows/python3.6.yaml.disabled From 7408f3ce0898dea20692929e949e336edd98dfe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:36 +0100 Subject: [PATCH 2012/2295] renamed python2.7.yaml to python2.7.yaml.disabled --- .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} (100%) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml.disabled similarity index 100% rename from .github/workflows/python2.7.yaml rename to .github/workflows/python2.7.yaml.disabled From 83b8b0506a21b3636ba4f75627f965c01fb0545f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:51:59 +0100 Subject: [PATCH 2013/2295] added python3.11.yaml --- .github/workflows/python3.11.yaml | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/python3.11.yaml diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..8e2d041ac --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} From 1cfb67773c00608d7fe6fc109f46a968747f63bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:53:20 +0100 Subject: [PATCH 2014/2295] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c6ae3495f..5b9392d62 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,11 @@ [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From d824e50952a1bb22fef5061544e37305f6b37623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 2015/2295] renamed pypy2.yaml to pypy2.yaml.disabled --- .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} (100%) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml.disabled similarity index 100% rename from .github/workflows/pypy2.yaml rename to .github/workflows/pypy2.yaml.disabled From 9091c212bd734a1eb605f29d3ecf516fcb12cbcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 2016/2295] renamed pypy3.yaml to pypy3.yaml.disabled --- .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} (100%) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml.disabled similarity index 100% rename from .github/workflows/pypy3.yaml rename to .github/workflows/pypy3.yaml.disabled From 8bfd91c7b4649060b5704ac3f092d62ae667ca11 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2017/2295] renamed ubuntu_14.04.yaml to ubuntu_14.04.yaml.disabled --- .../workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_14.04.yaml rename to .github/workflows/ubuntu_14.04.yaml.disabled From a4139b161723a350733d1ce22aab019e536a4c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2018/2295] renamed ubuntu_16.04.yaml to ubuntu_16.04.yaml.disabled --- .../workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_16.04.yaml rename to .github/workflows/ubuntu_16.04.yaml.disabled From 46ab2bf3cf7fbb872f451c9d0590a9b2dc8ce803 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2019/2295] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From d3e2d0be996674fe5ccedc47a9bce75b436532fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2020/2295] renamed ubuntu_20.04.yaml to ubuntu_20.04.yaml.disabled --- .../workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml rename to .github/workflows/ubuntu_20.04.yaml.disabled From 1f744ad1dbf1d66fddae98a6244c7f1f35ece41b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2021/2295] renamed ubuntu_github.yaml to ubuntu_github.yaml.disabled --- .../workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_github.yaml rename to .github/workflows/ubuntu_github.yaml.disabled From 72c62d30af56d108aa1554fb4884dd6abe635ba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:02:48 +0100 Subject: [PATCH 2022/2295] renamed ubuntu_18.04.yaml.disabled to ubuntu_18.04.yaml --- .../workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml.disabled rename to .github/workflows/ubuntu_18.04.yaml From 62d24176064c6b4dac09d71938d5f0d6e2b34c58 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:03:08 +0100 Subject: [PATCH 2023/2295] renamed ubuntu_20.04.yaml.disabled to ubuntu_20.04.yaml --- .../workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml.disabled b/.github/workflows/ubuntu_20.04.yaml similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml.disabled rename to .github/workflows/ubuntu_20.04.yaml From 209f3bfd5641f436862ffca3dece34d0bf81b3fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:05:48 +0100 Subject: [PATCH 2024/2295] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5b9392d62..32ee3ce53 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,6 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22) -[![Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22) [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) From e550318685f87e7e2c25b400e634175915a8debe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:06:12 +0100 Subject: [PATCH 2025/2295] renamed ubuntu_github.yaml.disabled to ubuntu_github.yaml --- .../workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu_github.yaml.disabled b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu_github.yaml.disabled rename to .github/workflows/ubuntu_github.yaml From c0e1bb251c0c9028b9f5b0907788e7766f346acf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:07:38 +0100 Subject: [PATCH 2026/2295] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From cbf21a97aabc95628d71f773f08229cab9ac8614 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:10:29 +0100 Subject: [PATCH 2027/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32ee3ce53..138775975 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) From a5ad66e4090134ecc51d4dbb869251e63f8f27c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 2028/2295] renamed centos.yaml to centos.yaml.disabled --- .github/workflows/{centos.yaml => centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos.yaml => centos.yaml.disabled} (100%) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml.disabled similarity index 100% rename from .github/workflows/centos.yaml rename to .github/workflows/centos.yaml.disabled From af02db5511f56a40b40a90702619c5dfd8f600a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 2029/2295] renamed centos7.yaml to centos7.yaml.disabled --- .github/workflows/{centos7.yaml => centos7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos7.yaml => centos7.yaml.disabled} (100%) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml.disabled similarity index 100% rename from .github/workflows/centos7.yaml rename to .github/workflows/centos7.yaml.disabled From 5b91ee48587426db040ac1afb31fbfef8f1fbeff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:41 +0100 Subject: [PATCH 2030/2295] renamed centos8.yaml to centos8.yaml.disabled --- .github/workflows/{centos8.yaml => centos8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos8.yaml => centos8.yaml.disabled} (100%) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml.disabled similarity index 100% rename from .github/workflows/centos8.yaml rename to .github/workflows/centos8.yaml.disabled From 79fa99d693ea366638e1ee6c38bd5dec65e4ee43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:28:28 +0100 Subject: [PATCH 2031/2295] updated README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 138775975..72b1c3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,6 @@ [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) [![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) -[![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) -[![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) -[![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) [![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) From 78ba04ce9d93aedfee00e09cb99c083884a53e1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:48:44 +0100 Subject: [PATCH 2032/2295] updated serf_event_handler.py --- serf_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serf_event_handler.py b/serf_event_handler.py index 455d7bee1..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': From f66248cc4545fe236327911857290247eba1a701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:49:41 +0100 Subject: [PATCH 2033/2295] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 2922c7f7f..d53a4bffa 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -109,7 +109,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: From 8446d976d4efcee77aad458c5086d16c86a74255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:53:59 +0100 Subject: [PATCH 2034/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72b1c3aab..e12942015 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) From d80f92c71f9aefed3a161dc9cf719766ad4c1eef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:05:15 +0100 Subject: [PATCH 2035/2295] renamed docker_pytools_centos.yaml to *_centos.yaml.disabled --- .../{docker_pytools_centos.yaml => *_centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{docker_pytools_centos.yaml => *_centos.yaml.disabled} (100%) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/*_centos.yaml.disabled similarity index 100% rename from .github/workflows/docker_pytools_centos.yaml rename to .github/workflows/*_centos.yaml.disabled From 930a2c341a40292160d34cfc827a6b8358f306a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:06:10 +0100 Subject: [PATCH 2036/2295] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e12942015..c301b354d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) -[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) From 2f629017ce6a08b0db62bff40c1a90f5f801ab53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 2037/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5e7dfbdc8..bccee5cd7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c +Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d From a109075a96e1d21f3dbe6cc21dd76b4a0706359e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 2038/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4515bb958..47048ee0c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b +Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b From 5bdc0c0db940cbc74019b7b37508c6943e0b65f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:33:03 +0100 Subject: [PATCH 2039/2295] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c301b354d..edb49e6b1 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,6 @@ [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) -[![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) -[![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) [git.io/pytools](https://git.io/pytools) From e2f37bf682f28263a993209c5e52bcef272025d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 2040/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bccee5cd7..995ad2de8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d +Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 From 8bb7bc784d8f0ad98dbefd691ee8bc460f727330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 2041/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 47048ee0c..7c0ba3afe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b +Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe From 383114f4538c6ccc9f2443d5c53d96e6f2c3d526 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:39:36 +0100 Subject: [PATCH 2042/2295] added ubuntu_22.04.yaml --- .github/workflows/ubuntu_22.04.yaml | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/ubuntu_22.04.yaml diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..b9bf17a95 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From 2769fba289020e9b94311b1f1bee3bcefc02f60e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 2043/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 995ad2de8..fe25b15b0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 +Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f From 6bfe3eb1a41357c469d3895b6739dfea28d7fab3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 2044/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7c0ba3afe..0a225da5c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe +Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a From 55827d1746db9186a49880e91d4f6d711058e7fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:55 +0100 Subject: [PATCH 2045/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fe25b15b0..2295778d8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f +Subproject commit 2295778d8039333abc3e159754641b348cf79d94 From 5491f93ea4295dd3a87c7541a5182b146baadfe7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:56 +0100 Subject: [PATCH 2046/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a225da5c..6e5e84d30 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a +Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 From a9c81105474dbfad815563507e21fa7b6f5d8785 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 2047/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2295778d8..37dede02a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2295778d8039333abc3e159754641b348cf79d94 +Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 From 3680e4c7ab65c6ef873bdf8151fe86efe68fd16a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 2048/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e5e84d30..5f2f181a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 +Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 From 8865ba8731144538dd0baba23d1ccb787c5988a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:27 +0100 Subject: [PATCH 2049/2295] added --unspace functionality --- center.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 4856ea0f7..b9cbac5f1 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' + class Center(CLI): @@ -56,7 +57,9 @@ def __init__(self): # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" #self.re_bound = re.compile(r'(\b)') self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') self.timeout_default = None def add_options(self): @@ -66,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -83,6 +91,11 @@ def space(self, line): line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -102,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() From ad1c2e05e9ebe50dcc9829209328c2fa9c3b9016 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:49 +0100 Subject: [PATCH 2050/2295] updated .pylintrc --- .pylintrc | 751 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 508 insertions(+), 243 deletions(-) diff --git a/.pylintrc b/.pylintrc index c20ac8f94..1c56d5de1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,386 +1,651 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= -# Specify a configuration file. -#rcfile= +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct method names. +method-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 +# Maximum number of return / yield for function / method body. +max-returns=6 -[ELIF] +# Maximum number of statements in function / method body. +max-statements=50 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -# -# avoids the following error: -# -# pylint -E ./check_zookeeper_version.py -# -# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) -# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) -# -ignored-classes=SQLObject,_socketobject +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From 64b326acbcea84535f033b706e6bfb5b289a5f32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 03:29:52 +0100 Subject: [PATCH 2051/2295] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index edb49e6b1..f064e78b0 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 0487abce888a1fdd84a18f853ef1871e17772147 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2052/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37dede02a..09a50e7e1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 +Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 From 493c6d1afb562ca7eca054bde3d48fd90b9586c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2053/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2f181a9..c3ab2659f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 +Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 From a0fed602600e01b0aa97e35e6824560e71de1bb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2054/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8d838daa6..0d8bd4872 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf +Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b From dd64db0434d294fc3bd53a2858e4da7920667009 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:04:10 +0100 Subject: [PATCH 2055/2295] added kics.config --- kics.config | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 kics.config diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..5fafd16d2 --- /dev/null +++ b/kics.config @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - pylib/ + - sql/ + - templates/ +#output-path: "results" From fee21a0c50c83186949d1ae3253f0882ae642054 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2056/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09a50e7e1..b2bcc4c93 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 +Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 From 131da36d872ec1e3b2013de15589703b594f5323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2057/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c3ab2659f..e137295ce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 +Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef From 0ba9caa0aa08f5e044c5e340b195ad008944fb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2058/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0d8bd4872..73d5f3b1f 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b +Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 From 5c929d22aa11b8553ec809a1b11172b933047ec2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 2059/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b2bcc4c93..9112b3d95 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 +Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 From 62046edaf17c3d87f6e7d1f281813e0ec1b2aaf2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 2060/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e137295ce..6a6b9cf72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef +Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 From 6f96ebf189863beae3e9612a6deb2743044dee24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 22:51:14 +0100 Subject: [PATCH 2061/2295] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f064e78b0..9c78001bb 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) From afe9d7a35dc370d604111a9e15c824e4d63f6fe2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 2062/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9112b3d95..a3750afbd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 +Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b From c9ac6b65bb38bbb6d5fa2375b5810d03fa9e46a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 2063/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6a6b9cf72..5f2c84abb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 +Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f From b2d609d7192f656094daf38192d4a396e5afee3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:49 +0100 Subject: [PATCH 2064/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 73d5f3b1f..590fd1816 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 +Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 From 0341c77f80096ce8a3c7171d5e92ced882d973e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:25:24 +0100 Subject: [PATCH 2065/2295] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9c78001bb..2ccdff67a 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,11 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 472331bac11a4d00a39cb865dcc41540e3cdbc8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:29:06 +0100 Subject: [PATCH 2066/2295] updated .pylintrc --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 1c56d5de1..b35a5fccc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -448,7 +448,10 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + missing-function-docstring, + super-with-arguments, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 93b2c4f3353810ae7a53f4b6707ec9e1c371b7a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:30:27 +0100 Subject: [PATCH 2067/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2c84abb..0f48c7572 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f +Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 From 2c42d10e551497e571f0297b36636f04bfb1025a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:45:22 +0100 Subject: [PATCH 2068/2295] updated .pylintrc --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index b35a5fccc..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -449,6 +449,7 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, + missing-class-docstring, missing-function-docstring, super-with-arguments, consider-using-f-string From 7a42f40f3dfa0d0a02658d82f630a59d3f5d5e8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:47:54 +0100 Subject: [PATCH 2069/2295] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index fb56e18cf..93ea98595 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -166,7 +166,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +197,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() From a282dda7d3eb16d5d8777db108bec344e3d90fff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 2070/2295] updated spark_avro_to_parquet.py --- spark_avro_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index 9300c4fd1..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 246d4df8036306782234ed6c1c4d47a7dee9d44e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 2071/2295] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 93ea98595..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -57,6 +57,7 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From dad1bce7ddc6cc5972ba5df65df8ff0138b1f659 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2072/2295] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 8e3b0da1c..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -49,6 +49,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') From 49c66be7fd0cc50d196ba7e6c1f0a503c14c1771 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2073/2295] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index d53a4bffa..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 7fb4da436c7016412fe3b7426ab4ea06b676e2e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2074/2295] updated spark_json_to_parquet.py --- spark_json_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index 2ed935917..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 86727ce5e22f53fa4d84627865c4cad83dddf70e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2075/2295] updated spark_parquet_to_avro.py --- spark_parquet_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index 0186981f5..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From ec22d9ca18fbd0ec75d1137a2ae11c8001247286 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:50 +0100 Subject: [PATCH 2076/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3750afbd..1b0b1149f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b +Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 From f81d2e16df704a4fee44f20a3a48658b0e1f7c17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 2077/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f48c7572..e53f962a7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 +Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb From 1643e01fddfd93d2c301a9a6ca925c1192389ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 2078/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 590fd1816..3a74368ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 +Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f From dbc5cec14dea46ef3f851d56988ab9173f8ace9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:01 +0100 Subject: [PATCH 2079/2295] updated Makefile --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a67a168a9..13e4c9b27 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -118,15 +118,15 @@ python: pylib @# only install pip packages not installed via system packages @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @#$(SUDO_PIP) $(PIP) install -r requirements.txt - @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3"; fi - PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @@ -154,7 +154,7 @@ python: pylib @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @PIP=$(PIP) bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -216,7 +216,7 @@ test: .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build From bf4934fe8873bc1153ac2fef4f796a112fc93483 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:38 +0100 Subject: [PATCH 2080/2295] updated all.sh --- tests/all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 33452eb7c..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; # shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. "$srcdir/../bash-tools/check_all.sh" +. "$srcdir/../bash-tools/checks/check_all.sh" #tests/test_yamllint.sh @@ -41,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh From 9a1f9be8ea238a1bc7cb04291f1eb743308bc09b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:47 +0100 Subject: [PATCH 2081/2295] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 1f34f863a..abb7e9d21 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -68,7 +68,7 @@ for filename in $file_list; do echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" From 7251c68560d2f1f605475a8fb0df98e28c18353f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:02:38 +0100 Subject: [PATCH 2082/2295] updated sonar-project.properties --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 270ec8ac7..7189b1a16 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -21,7 +21,7 @@ sonar.projectDescription=Python / Jython Tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions sonar.sources=. From 754947957d53ef5ed572bdcbb2b91a6f4f4b583c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:59:10 +0100 Subject: [PATCH 2083/2295] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ccdff67a..56a02a41e 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,10 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [Jenkins](https://github.com/HariSekhon/Jenkins) - Advanced Jenkinsfile & Jenkins Groovy Shared Library + +- [GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions) - GitHub Actions master template & GitHub Actions Shared Workflows library + - [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments From 01af37704aed0bcbe5eeb78d0bbcd3e849358b63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:37:00 +0100 Subject: [PATCH 2084/2295] added trivy.yaml --- .github/workflows/trivy.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..5c9475308 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +--- +name: Trivy + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy Filesystem Scan + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 0d560dd9f6634ad3ef668f2aa8ee52ab20d0f204 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:47:26 +0100 Subject: [PATCH 2085/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56a02a41e..aeab7f613 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) -[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) From 585e9050a905151a82746a4a8bfe42b993bf7ed6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:08:50 +0100 Subject: [PATCH 2086/2295] updated trivy.yaml --- .github/workflows/trivy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 5c9475308..8af2aeb39 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # @@ -54,7 +54,7 @@ jobs: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' - name: Trivy Filesystem Scan + name: Trivy uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master with: debug: ${{ github.event.inputs.debug }} From 92b23f826de7fcdab498eb6b9e0ac26b306f09d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:41:06 +0100 Subject: [PATCH 2087/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aeab7f613..a41120672 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) [![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) From 9fe0d3ca28260ec2c4414a6f77c6ebbd0e5c390f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 07:15:53 +0100 Subject: [PATCH 2088/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a41120672..36f322455 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ## Related Repositories -- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 800+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 1000+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From b8c21da103ec580fb9b26ade93ad3eeb664cd2db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 May 2023 04:18:31 +0100 Subject: [PATCH 2089/2295] added grype.yaml --- .github/workflows/grype.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/grype.yaml diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..6a321f86e --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From a8c29b46dd6d8b0659fb64bbbc41c1d2cd6666e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 May 2023 02:06:58 +0100 Subject: [PATCH 2090/2295] updated center.py --- center.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/center.py b/center.py index b9cbac5f1..379353e68 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class Center(CLI): @@ -104,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): From f9abc5f8321e73b34e0be59079823ce850b50362 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:24 +0100 Subject: [PATCH 2091/2295] removed codeship.yml --- codeship.yml | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml deleted file mode 100644 index f466bd6a7..000000000 --- a/codeship.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# ============================================================================ # -# C o d e S h i p -# ============================================================================ # - -# LEGACY - CloudBees has removed free plans as of 1st March 2023 - -# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this - -# https://github.com/painless-software/codeship-yaml - -# Requires setting up the CodeShip commands like so: -# -# pip install codeship-yaml -# codeship-yaml -# -# or seaparately in sections: -# -# Project Settings > Test Settings > Setup Commands: -# -# pip install codeship-yaml -# codeship-yaml install -# -# Project Settings > Test Settings > Test Commands: -# -# codeship-yaml before_script script -# -# Project Settings > Deployment > (branch name) -# -# codeship-yaml after_success - ---- -install: - # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - - sudo rm -fv -- /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - - sudo rm -fv -- /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list - - sudo rm -fv -- /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - - make -#before_script: -# - somecommand -script: - - make test -#after_success: -# - echo "Now we can deploy" From c4c513b66715f87bbed88320deda71205f84ce96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:32 +0100 Subject: [PATCH 2092/2295] removed wercker.yml --- wercker.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index c27be30ed..000000000 --- a/wercker.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# https://devcenter.wercker.com/reference/wercker-yml/ - -box: debian - -build: - steps: - - script: - name: ci bootstrap - code: setup/ci_bootstrap.sh - - script: - name: init - code: make init - - script: - name: build - code: make ci - - script: - name: test - code: make test From 116cdfda1b85dbd8803e21a3e126c1d4d58d8ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:38 +0100 Subject: [PATCH 2093/2295] removed .werckerignore --- .werckerignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore deleted file mode 100644 index ff8fb8247..000000000 --- a/.werckerignore +++ /dev/null @@ -1 +0,0 @@ -**/.md From 5ba24dc535c907d2d574d652da6517e3ec2cf75a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:08 +0100 Subject: [PATCH 2094/2295] moved .teamcity.vcs.json to teamcity/ --- .teamcity.vcs.json => teamcity/.teamcity.vcs.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.json => teamcity/.teamcity.vcs.json (100%) diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json similarity index 100% rename from .teamcity.vcs.json rename to teamcity/.teamcity.vcs.json From 1730b699555891aff16137749cfbd88f7559f21b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 2095/2295] moved .teamcity.vcs.oauth.json to teamcity/ --- .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.oauth.json rename to teamcity/.teamcity.vcs.oauth.json From 21d86fe66caa9b86030bedd0155287ad263ba17f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 2096/2295] moved .teamcity.vcs.ssh.json to teamcity/ --- .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json (100%) diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json similarity index 100% rename from .teamcity.vcs.ssh.json rename to teamcity/.teamcity.vcs.ssh.json From 637c5ade2863627f66f26bd6ee354416c698e69d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2097/2295] moved .concourse.yml to cicd/ --- .concourse.yml => cicd/.concourse.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .concourse.yml => cicd/.concourse.yml (100%) diff --git a/.concourse.yml b/cicd/.concourse.yml similarity index 100% rename from .concourse.yml rename to cicd/.concourse.yml From 424a095fedc3298d2ce34a63f6a4296b2acb8140 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2098/2295] moved .gocd.yml to cicd/ --- .gocd.yml => cicd/.gocd.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .gocd.yml => cicd/.gocd.yml (100%) diff --git a/.gocd.yml b/cicd/.gocd.yml similarity index 100% rename from .gocd.yml rename to cicd/.gocd.yml From 2451e74fd432d4cdac907970369e2c9059608f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2099/2295] moved buildspec.yml to cicd/ --- buildspec.yml => cicd/buildspec.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename buildspec.yml => cicd/buildspec.yml (100%) diff --git a/buildspec.yml b/cicd/buildspec.yml similarity index 100% rename from buildspec.yml rename to cicd/buildspec.yml From fca148ae783746b8452a26818344156c5a1bcdae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2100/2295] moved cloudbuild.yaml to cicd/ --- cloudbuild.yaml => cicd/cloudbuild.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cloudbuild.yaml => cicd/cloudbuild.yaml (100%) diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to cicd/cloudbuild.yaml From caf90ee899ba2b558bdbab7d28c87ae28c5da7ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:41:46 +0100 Subject: [PATCH 2101/2295] removed shippable.yml --- shippable.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 1fa06bc06..000000000 --- a/shippable.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# http://docs.shippable.com/platform/workflow/config/ - -# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ - -language: none - -branches: - only: - - master - -build: - ci: - # workaround to broken repos - # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB - # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 - # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - - rm -fv -- /etc/apt/sources.list.d/cassandra.sources.list* - - rm -fv -- /etc/apt/sources.list.d/yarn.list* - # Basho repo is giving a '402 payment required' error - # https://github.com/Shippable/support/issues/5172 - - rm -fv -- /etc/apt/sources.list.d/basho_riak.list - #- shippable_retry make - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_pull_request: never From 7d04cce80786cc6c4f278ce4a773f7d6a6f69ae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:01 +0100 Subject: [PATCH 2102/2295] updated .appveyor.yml --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f4b36fba0..c459fd55a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu From 00b3cf9ae996e7b5a8f5bc7577adfd9b420b9381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:02 +0100 Subject: [PATCH 2103/2295] updated pipeline.yml --- .buildkite/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93f047c02..eb0d641fd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: From 526798d5606da7d44e405fb9931b9d27c9c39b71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:03 +0100 Subject: [PATCH 2104/2295] updated config.yml --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dbde36b7..09026fc6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml From 126ad67b87f9f2ada233929556a37d3ee479126e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:04 +0100 Subject: [PATCH 2105/2295] updated .cirrus.yml --- .cirrus.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index d544aeb85..6916e2577 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + # https://cirrus-ci.org/guide/writing-tasks/ container: From 243482c43161216eb1c278c64a0451f731347385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:05 +0100 Subject: [PATCH 2106/2295] updated .drone.yml --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4d36c9db2..d105ab57a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ From 84d0b04dff32333bb7da8058de7c36b98bab6015 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:06 +0100 Subject: [PATCH 2107/2295] updated .gitlab-ci.yml --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477b32c3e..625220b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + # https://docs.gitlab.com/ee/ci/yaml/README.html +#include: '.gitlab/*.y*ml' + image: ubuntu:18.04 job: From c21088d44d170cf28533c2af5633c753b3a493ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:15:38 +0100 Subject: [PATCH 2108/2295] updated sonar-project.properties --- sonar-project.properties | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 7189b1a16..9b1539430 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # # https://github.com/HariSekhon/DevOps-Python-tools # @@ -12,21 +12,30 @@ # https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +sonar.exclusions=**/tests/** From 2c93d263101b61483ad2189406aefe87efe53ecf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:39 +0100 Subject: [PATCH 2109/2295] updated sonar-project.properties --- sonar-project.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 9b1539430..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -38,4 +38,5 @@ sonar.sources=. sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/** +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* From 3b6c0b0324ba01c6596adb589c86ad6790225fbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:50 +0100 Subject: [PATCH 2110/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e86c4f36e..761ef0697 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 From 7e975e2cda81ac8b0177401bc8766caf7db9bf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:51 +0100 Subject: [PATCH 2111/2295] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 36f322455..35a30a739 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,8 +55,8 @@ [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 97a95b048ddff13a76f31a2687f62a8c3f4ee10f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:52 +0100 Subject: [PATCH 2112/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76ad8f1f0..e26e7f760 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + # https://aka.ms/yaml trigger: From e7b73f1e14052b5e152d8934fddd83e11929db02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:53 +0100 Subject: [PATCH 2113/2295] updated buddy.yml --- buddy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buddy.yml b/buddy.yml index f901a6425..e0b57451f 100644 --- a/buddy.yml +++ b/buddy.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + # https://buddy.works/docs/yaml/yaml-schema --- From 3f82815fe59aae0a425f870296787d9206c161c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:59 +0100 Subject: [PATCH 2114/2295] updated .concourse.yml --- cicd/.concourse.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml index 42b806f26..92847121e 100644 --- a/cicd/.concourse.yml +++ b/cicd/.concourse.yml @@ -13,6 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html resources: - name: github icon: github-circle @@ -25,8 +33,7 @@ resources: # source: # interval: 1d -# https://concourse-ci.org/golang-library-example.html - +# https://concourse-ci.org/jobs.html jobs: - name: build public: false From 3851ded922052a5aa98677a3045fcbca03a13aa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 2115/2295] updated .gocd.yml --- cicd/.gocd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml index f3ed9fd19..65583a818 100644 --- a/cicd/.gocd.yml +++ b/cicd/.gocd.yml @@ -12,6 +12,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G o C D +# ============================================================================ # + # https://github.com/tomzo/gocd-yaml-config-plugin#setup # https://docs.gocd.org/current/configuration/configuration_reference.html From f7dda886aad2cdcd0374877ca6b5ab9dae4bd2d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 2116/2295] updated codefresh.yml --- codefresh.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codefresh.yml b/codefresh.yml index 5326f6e51..7cddcaf7c 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + # https://codefresh.io/docs/docs/codefresh-yaml/ version: "1.0" From 0865b14f7edd4a67a348af0139713a706fa67e3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:01 +0100 Subject: [PATCH 2117/2295] updated kics.config --- kics.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kics.config b/kics.config index 5fafd16d2..51ac5712e 100644 --- a/kics.config +++ b/kics.config @@ -30,7 +30,15 @@ log-file: true exclude-paths: # ignore submodules - handle them in the source repos only - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ - pylib/ + - spotify-tools/ - sql/ + - sql-keywords/ - templates/ + - terraform-templates/ #output-path: "results" From 0ba05249f4a515b998110738105078ef6cb4d003 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:02 +0100 Subject: [PATCH 2118/2295] updated gocd_config_repo.json --- setup/gocd_config_repo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 4c1639b4d..ce63bf63d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -12,7 +12,7 @@ "configuration": [ { "key": "file_pattern", - "value": "*.gocd.y*ml" + "value": "cicd/*.gocd.y*ml" } ], "rules": [ From f605276d04c77181fa4e8cf71c85cf6002752a35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:05 +0100 Subject: [PATCH 2119/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b0b1149f..36e0eabe2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 +Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 From 37b79941c2329f2de041cab714d08851c1eaa5f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2120/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e53f962a7..af52e8795 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb +Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 From 0d7d8060f515735afca206920e275643934e4867 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2121/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d4eedc0e2..b96c184fd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 +Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 From d1ef74ffebc27e32ec9afeb5bca7debd4fedb64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2122/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3a74368ea..d58140c42 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f +Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 From a3634ecf1e18a624af24814f4540cba15623a642 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:32 +0100 Subject: [PATCH 2123/2295] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6b9eaf3b9..4dc3a09c3 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,8 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners on: push: From 79fdd3ef11e57c87b703255fc61789aedd083080 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:33 +0100 Subject: [PATCH 2124/2295] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 4d4ebd577..c7ff16fd4 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync From 6900c0b4ca31f052e94e7caffb77c16ec8dd7938 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:34 +0100 Subject: [PATCH 2125/2295] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 00479ecb7..94398083f 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,6 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR From 02e4b41f7117802742d8af57107da9aa09677982 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 2126/2295] updated json.yaml --- .github/workflows/json.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 6ed4b92d1..f68e309e5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON From 77f1608b618c75c87ad51006dfddba709aabdf37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 2127/2295] updated kics.yaml --- .github/workflows/kics.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 399cc40f0..033f4331e 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -11,6 +11,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# K i c s +# ============================================================================ # + --- name: Kics From 70a101b1ae1095ba49d05a182d3a77a7eb352f73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 2128/2295] updated shellcheck.yaml --- .github/workflows/shellcheck.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 98e53bfe4..753a6a495 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + --- name: ShellCheck From 55a9f677eb8afdb95879e413b15da0b3d5543b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 2129/2295] updated trivy.yaml --- .github/workflows/trivy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 8af2aeb39..26f01fa94 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -15,6 +15,8 @@ # T r i v y # ============================================================================ # +# Scan files in the local repo + --- name: Trivy From b5f76c8c6c099ca81c0e0d6885d41f1195d85379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 2130/2295] updated validate.yaml --- .github/workflows/validate.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 5d924b77b..94a8c007a 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation From 88ac6b6661928c5f272515fe0700576d04b6f279 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 2131/2295] updated xml.yaml --- .github/workflows/xml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index f4d3b7035..8b07ed256 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + --- name: XML From 2c992b85d84bb9dd9ebd7ea2e3ab57dfafe13ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:38 +0100 Subject: [PATCH 2132/2295] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index bee8be84c..d252f67d2 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML From a2a575ec7dec99394aab6b8572f42953129e3e43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:49 +0100 Subject: [PATCH 2133/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36e0eabe2..815d75082 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 +Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae From 202d2ed1a86e2bd72f131fdd262eb6b88a846132 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2134/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index af52e8795..ae8aa8753 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 +Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 From 25abbfdedc8b9832c507f970f7c1a8ef29643a27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2135/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b96c184fd..e11486ebd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 +Subproject commit e11486ebd436802d22d1b88bda00c0a9eb3d41b6 From 24407a2473022336a734c95d262203780f75df87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2136/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d58140c42..3af953c9a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 +Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 From 82da083e55bda834adf1802587856c6ff5b8b95c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:01 +0100 Subject: [PATCH 2137/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 815d75082..38901a555 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae +Subproject commit 38901a555d2ce114f0e7df6d4a31a3fdc1a3dc41 From 4b432be8807c49caa2682b2f04ad492d403f2f0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 2138/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae8aa8753..aa20abd72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 +Subproject commit aa20abd72ceef27723ad8c461f4d9321ad12e779 From e14414287476521c71efa1ca3cb1095d2886ca62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 2139/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3af953c9a..15d1e3bc8 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 +Subproject commit 15d1e3bc8a594171b3c70ce0bf04979eceb8630a From baec6ac43c0a3bf03cd7d78bb35d9972d470c922 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 17:46:12 +0100 Subject: [PATCH 2140/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a30a739..eb8e5fe12 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, ArgoCD, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... From b5db3cb80db2ddc5dcc980a844939766452dc308 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 2174/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da278a058..1438f07bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da278a05823c78709d831a4e6ba9cce4e5a481a7 +Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 From 8e50ccbc1c4ec668a1cea00152de7c700b584d24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 2175/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..6f9e99389 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit 6f9e993896910c320c571918a635dc049e757e73 From a2d58fac6ac19b0d1b5f663aab5939eefe2b9f28 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:33:56 +0400 Subject: [PATCH 2176/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1438f07bd..b69edeada 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 +Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 From 39a4591fa22b1a6c0ffe94e0f8451fa3a5a60f1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 May 2024 21:55:51 +0400 Subject: [PATCH 2177/2295] added .envrc --- .envrc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6fa553de2 --- /dev/null +++ b/.envrc @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r e n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +# See more .envrc files in: +# +# https://github.com/HariSekhon/DevOps-Bash-tools +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export COMPOSE_PROJECT_NAME="pytools" + +venv="$PWD/venv" + +if [ -f "$venv/bin/activate" ]; then + echo + echo "Local virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo +fi + +# read .env too +#dotenv From a0d2bc4abf677d283f992db3bab72461e888943f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:19:34 +0200 Subject: [PATCH 2178/2295] updated .envrc --- .envrc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.envrc b/.envrc index 6fa553de2..96d3b5c0f 100644 --- a/.envrc +++ b/.envrc @@ -35,18 +35,18 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" -venv="$PWD/venv" - -if [ -f "$venv/bin/activate" ]; then - echo - echo "Local virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" - - # shellcheck disable=SC1091 - source "$venv/bin/activate" - echo -fi +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo + fi +done # read .env too #dotenv From 0f9f99ff568ae21d903ceaaf0bf77b65c93632cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:20:41 +0200 Subject: [PATCH 2179/2295] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 96d3b5c0f..2aa9a4379 100644 --- a/.envrc +++ b/.envrc @@ -35,6 +35,7 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default for venv in "$PWD/venv" "$HOME/venv"; do if [ -f "$venv/bin/activate" ]; then echo From 92451e77812ddd6ace3c3ac401887fbeb2e1ad89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:30 +0200 Subject: [PATCH 2180/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b69edeada..4cf683ab7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 +Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 From eef5f927cf6bc15c6d438706a37567529347dd7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 2181/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 99f5c866a..804fcd30f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 99f5c866aa5986c62d45de3b8da7043a28a63321 +Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 From 9616644dba9bf8d67673bcf02ae431af2395579d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 2182/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f9e99389..a43798fe9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f9e993896910c320c571918a635dc049e757e73 +Subproject commit a43798fe986928bffe4fe5952efdce408d012562 From 7dd10775f06d708e16930d91519297addaa33519 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2183/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4cf683ab7..8e8cf26f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 +Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 From aa3ef778e851dd980c962b4fafe2600dffd50123 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2184/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 804fcd30f..c88505e98 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 +Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d From 749724f6c95f2baba13490258a14b79b45c79f2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2185/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..c2521be52 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit c2521be525b4047222adb18263e5714e4171c483 From d8cbf436f9842d01a3bda55fdb095b2a4209b50d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:36:09 +0200 Subject: [PATCH 2186/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c88505e98..545ce8b83 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d +Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d From 21f534993310e30479c38661de27c0ffc350946e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:38:47 +0200 Subject: [PATCH 2187/2295] updated .envrc --- .envrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envrc b/.envrc index 2aa9a4379..9073c82fd 100644 --- a/.envrc +++ b/.envrc @@ -14,7 +14,7 @@ # # ============================================================================ # -# D i r e n v +# P y t h o n D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present From 82154119348fccba6d13f674c336bdc127c4ac44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 2188/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8e8cf26f2..6d8f6ebc0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 +Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce From 899a4cea79947f03e8e0862345914ddf47e68bee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 2189/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545ce8b83..76e9e28d1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d +Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa From a63287671e5471c97794e9c19f41b676d3d12e81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2190/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d8f6ebc0..9f1a8c533 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce +Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f From 18cc287778458c4bfcaea0c12fca330302e6982c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2191/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76e9e28d1..e2982d2f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa +Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c From 1f6f01941062e293ee1c8898e884bafca470dccc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2192/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 39d0eccb4..8597f692e 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 39d0eccb4c160fcee9f09df9bb53c911ca2a387f +Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 From e805b2ab2933e6a05cfb0fbd0f6dfbabf5e50c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2193/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c2521be52..5174a5073 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c2521be525b4047222adb18263e5714e4171c483 +Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 From 86d645a4f49915bd3032fcc9fee866f0a437eb94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:40:47 +0200 Subject: [PATCH 2194/2295] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 9073c82fd..51de39abb 100644 --- a/.envrc +++ b/.envrc @@ -46,6 +46,7 @@ for venv in "$PWD/venv" "$HOME/venv"; do # shellcheck disable=SC1091 source "$venv/bin/activate" echo + break fi done From c6fcf3037124d84f6f0b9ce569993fe41eb76181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 2195/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9f1a8c533..f32bb01be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f +Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 From 15816a21b449f0af739eb5985b61fe7ab9f9f475 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 2196/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e2982d2f4..29fbc0f2d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c +Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 From 959c4ce33775ecf588e4ad5794d1c340752510a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:33:25 +0200 Subject: [PATCH 2197/2295] updated .flake8 --- .flake8 | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 From 06d5e9469a49cf33c0c618ac5e05fbd4343dd4c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2198/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f32bb01be..057b80266 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 +Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 From 49075c27e13a166fc5ac4dfd5582dfc9c936b928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2199/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 29fbc0f2d..fdb270866 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 +Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 From cd4e61aa6c5d4ff4181342e5d8f8581a1b135493 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2200/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5174a5073..d94994282 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 +Subproject commit d94994282274abf4279444fd85c0674a382e6696 From ec92f71bad8c41778516345018fd6295861aa67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:17:37 +0300 Subject: [PATCH 2201/2295] added .envrc-python --- .envrc-python | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .envrc-python diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv From bae58f9cbadd09648d416c363f73e5ae1d5dd582 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:19:10 +0300 Subject: [PATCH 2202/2295] updated .envrc --- .envrc | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/.envrc b/.envrc index 51de39abb..7e481411c 100644 --- a/.envrc +++ b/.envrc @@ -14,41 +14,150 @@ # # ============================================================================ # -# P y t h o n D i r E n v +# D i r E n v # ============================================================================ # -# .envrc to auto-load the virtualenv inside the 'venv' directory if present - # https://direnv.net/man/direnv-stdlib.1.html -# See more .envrc files in: -# -# https://github.com/HariSekhon/DevOps-Bash-tools +# See Also: # # .envrc-aws # .envrc-gcp # .envrc-kubernetes +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" -export COMPOSE_PROJECT_NAME="pytools" +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # -# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default -for venv in "$PWD/venv" "$HOME/venv"; do - if [ -f "$venv/bin/activate" ]; then - echo - echo "Virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" +# Automatically install Pre-Commit Git hooks if not already present - # shellcheck disable=SC1091 - source "$venv/bin/activate" +if [ -f .pre-commit-config.yaml ]; then + if [ -d .git ]; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install + fi + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# P y t h o n , A W S , G C P , T e r r a f o r m +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi echo - break + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" fi +} + +#load_if_exists ~/.envrc + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" done +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # + +echo # read .env too #dotenv From 13b3cde4939345bbb49086f16b627056958b2329 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:51:24 +0300 Subject: [PATCH 2203/2295] added .pre-commit-config.yaml --- .pre-commit-config.yaml | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f3c5b1608 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,72 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/Templates +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + language: ruby + files: \.(md|mdown|markdown)$ From 5c145c7d2e6459c61a9a55c0f6f7047e009ccbda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:57:50 +0300 Subject: [PATCH 2204/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3c5b1608..2b154867f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https///github.com/HariSekhon/Templates +# https///github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # From c8c6623a14f7482ab9b222f63074f0bcbb55e657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 20:33:29 +0300 Subject: [PATCH 2205/2295] updated .envrc --- .envrc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.envrc b/.envrc index 7e481411c..da56f607a 100644 --- a/.envrc +++ b/.envrc @@ -52,11 +52,11 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if [ -f .pre-commit-config.yaml ]; then - if [ -d .git ]; then - if ! [ -f .git/hooks/pre-commit ]; then - pre-commit install - fi +if [ -f .pre-commit-config.yaml ] && + [ -d .git ] && + type -P pre-commit &>/dev/null; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install fi fi From 9a5c385d480ecf1e05aacf4e0e753af684f9e262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 00:12:38 +0300 Subject: [PATCH 2206/2295] updated .editorconfig --- .editorconfig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 73c19b202..462f53206 100644 --- a/.editorconfig +++ b/.editorconfig @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf From f410868fe774c96117e790f2bb7f6f2bd3fd9af6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 01:49:04 +0300 Subject: [PATCH 2207/2295] updated .envrc --- .envrc | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index da56f607a..920b4e16d 100644 --- a/.envrc +++ b/.envrc @@ -98,7 +98,7 @@ export COMPOSE_PROJECT_NAME="DevOps-Python-tools" #export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # ============================================================================ # -# P y t h o n , A W S , G C P , T e r r a f o r m +# Load External Envrc Files If Present # ============================================================================ # # XXX: safer to bring all these external .envrc inline if you're worried about changes @@ -124,8 +124,15 @@ load_if_exists(){ fi } +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) #load_if_exists ~/.envrc +# ============================================================================ # +# P y t h o n +# ============================================================================ # + #.envrc-aws \ #.envrc-gcp \ #.envrc-terraform \ @@ -136,14 +143,26 @@ for envrc in \ load_if_exists "$envrc" done +# ============================================================================ # +# A W S +# ============================================================================ # + if [[ "$PWD" =~ /aws/ ]]; then load_if_exists .envrc-aws fi +# ============================================================================ # +# G C P +# ============================================================================ # + if [[ "$PWD" =~ /gcp/ ]]; then load_if_exists .envrc-gcp fi +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then load_if_exists .envrc-terraform fi From 283a9a3ca15ea2293c9ae2ed08a9ac77aa7d0bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 12:06:01 +0300 Subject: [PATCH 2208/2295] updated .envrc --- .envrc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 920b4e16d..15bc469ee 100644 --- a/.envrc +++ b/.envrc @@ -52,10 +52,23 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present +if ! type -P pre-commit &>/dev/null && + uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now..." + echo + brew install pre-commit + echo +fi + if [ -f .pre-commit-config.yaml ] && - [ -d .git ] && - type -P pre-commit &>/dev/null; then - if ! [ -f .git/hooks/pre-commit ]; then + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo pre-commit install fi fi From f09fdd3279da8103197fc4be455be86eea23e2df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 14 Aug 2024 23:17:32 +0200 Subject: [PATCH 2209/2295] updated .envrc --- .envrc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.envrc b/.envrc index 15bc469ee..7555ecba9 100644 --- a/.envrc +++ b/.envrc @@ -52,14 +52,20 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if ! type -P pre-commit &>/dev/null && - uname -s | grep -q Darwin && - type -P brew &>/dev/null; then - echo - echo "Pre-commit is not installed - installing now..." - echo - brew install pre-commit - echo +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi fi if [ -f .pre-commit-config.yaml ] && @@ -188,6 +194,8 @@ if [ -f "$srcdir/.envrc-kubernetes" ]; then load_if_exists .envrc-kubernetes docker-desktop fi +# ============================================================================ # +# . E n v # ============================================================================ # echo From d31b9e28d672eedc9624cfc2890daf982fd6acd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 15 Aug 2024 16:35:10 +0200 Subject: [PATCH 2210/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 503347f5d..cc918aa8d 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + [![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) From f3c5690d32c077c52b108ca1f83f88fbae06cfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:09 +0200 Subject: [PATCH 2211/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b154867f..df14fb7d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,5 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl + args: [-s, .markdownlint.rb] language: ruby files: \.(md|mdown|markdown)$ From 4546c7688be6a4e0a12b333b8fa123c0aae7a91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:10 +0200 Subject: [PATCH 2212/2295] added .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb new file mode 100644 index 000000000..e69de29bb From db58f5309bcf6fe5d145bb65afdc501b0c1494d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:18 +0200 Subject: [PATCH 2213/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df14fb7d9..9e9c1c291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl - args: [-s, .markdownlint.rb] + args: [-s, .mdl.rb] language: ruby files: \.(md|mdown|markdown)$ From 0839b2b170aa75b94c39263577660b3cac5e25ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 2214/2295] added .mdlrc --- .mdlrc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mdlrc diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file From c8ec774ef167b959685256791528eba64dbd8b32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 2215/2295] added .mdl.rb --- .mdl.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .mdl.rb diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..db6a3373f --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +#exclude_rule 'MD039' +#exclude_rule 'MD056' From cfa61d4d041dba2306dfdfbf568226209d656f56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:36:37 +0200 Subject: [PATCH 2216/2295] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index ab20cfa8d..0b52378ff 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -52,7 +52,7 @@ retry(){ if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) - #retry "$srcdir/install_homebrew.sh" + #retry "$srcdir/../install/install_homebrew.sh" if command -v brew 2>&1; then # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do From 63bde7fef5cf084fa7fe4a0a212d60d275f46ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2217/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 057b80266..e7c73c3c4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 +Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 From 16a294f584f532cde95b2df7401cc50234bdd3d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2218/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fdb270866..092819070 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 +Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 From c91a96c1ebd6a698d568d6a36d758872da9804b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2219/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8597f692e..ace6d6400 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 +Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a From b53766eb78df127afdcc81e3dde0a50a8f9dabe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:20 +0200 Subject: [PATCH 2220/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d94994282..b8d565217 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d94994282274abf4279444fd85c0674a382e6696 +Subproject commit b8d565217a111514a2e061cec896f258712985fb From 82e9390f13636322f39891aa5cc372a4496428c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:15:58 +0200 Subject: [PATCH 2221/2295] updated README --- README.md | 196 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 116 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index cc918aa8d..4fd1177dc 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +### Knowledge -- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -- [Nagios Plugin Kafka](https://github.com/HariSekhon/Nagios-Plugin-Kafka) - Kafka API pub/sub Nagios Plugin written in Scala with Kerberos support + -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +### DevOps Code -- [Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) - IT Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, AWS, GCP etc. +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization -- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) -### Stargazers over time +### CI/CD -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -[git.io/python-tools](https://git.io/python-tools) +### DBA - SQL -[git.io/pytools](https://git.io/pytools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) + +### Misc + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + +![](https://hit.yhype.me/github/profile?user_id=2211051) + + From e27764e4a9ed13170d3585259ea6267be3b7f62d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:17:33 +0200 Subject: [PATCH 2222/2295] updated README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4fd1177dc..5fa8938af 100644 --- a/README.md +++ b/README.md @@ -534,14 +534,17 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -### Misc +### Templates +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Misc + [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) From 6259d5a9e744abde66b0ff91c15b194e8104b075 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:19:58 +0200 Subject: [PATCH 2223/2295] updated .mdl.rb --- .mdl.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/.mdl.rb b/.mdl.rb index db6a3373f..f8f9b004c 100644 --- a/.mdl.rb +++ b/.mdl.rb @@ -25,5 +25,6 @@ #exclude_rule 'MD031' #exclude_rule 'MD032' exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines #exclude_rule 'MD039' #exclude_rule 'MD056' From e4eac61b95c830eb6ee80027214d72e9be9dcd6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:30:55 +0200 Subject: [PATCH 2224/2295] updated README --- README.md | 176 +++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5fa8938af..955eb8cb6 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Cloud & Big Data Contractor, United Kingdom [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/)
*(you're welcome to connect with me on LinkedIn)* -**Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries** +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** ## Quick Start @@ -163,14 +163,14 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git ### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -179,115 +179,115 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - - see also ```validate_*.py``` further down for all these formats and more + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table - - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -326,7 +326,7 @@ Download the DevOps Python Tools and Pylib git repos as zip files: -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. ```shell unzip devops-python-tools-master.zip @@ -450,7 +450,7 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). ### Testing @@ -476,9 +476,9 @@ You might also be interested in the following really nice Jupyter notebook for H -### Stargazers over time +## Star History -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) [git.io/python-tools](https://git.io/python-tools) From 73ff2ef8fe71271639bea252a671c6a70fbf5830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:32:22 +0200 Subject: [PATCH 2225/2295] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 955eb8cb6..8c3ad36f1 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ Environment variables are supported for convenience and also to hide credentials - [Hadoop](http://hadoop.apache.org/) HDFS: - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output From 94d58f49cb50bbe8ef645d3a785914cd6231b6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 20:48:32 +0200 Subject: [PATCH 2226/2295] updated README.md --- README.md | 253 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8c3ad36f1..f9fdde42d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ cd pytools make ``` -To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: ```shell make anonymize.pyc @@ -165,12 +166,15 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -181,27 +185,57 @@ Environment variables are supported for convenience and also to hide credentials - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) - - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` - - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter @@ -213,7 +247,13 @@ Environment variables are supported for convenience and also to hide credentials - `xml_to_json.py` - XML to JSON converter - `json_to_xml.py` - JSON to XML converter - `json_to_yaml.py` - JSON to YAML converter - - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: @@ -223,69 +263,119 @@ Environment variables are supported for convenience and also to hide credentials - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table - - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations - - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables - - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables - - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables - - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - `pig_udfs.jy` - Pig Jython UDFs for Hadoop -- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: @@ -297,14 +387,20 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions #### Python VirtualEnv localized installs -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. ### Manual Setup @@ -337,7 +433,8 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. @@ -355,9 +452,11 @@ sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt #### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): ```shell bash-tools/install/install_homebrew.sh @@ -373,7 +472,8 @@ If failing to build an OpenSSL lib dependency, just prefix the build command lik sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: ```shell sudo pip install --upgrade pip==9.0.1 @@ -398,7 +498,8 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. @@ -422,7 +523,10 @@ Then add the Jython install bin directory to the $PATH or specify the full path ### Configuration for Strict Domain / FQDN validation -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). ### Python SSL certificate verification problems @@ -433,7 +537,8 @@ If you end up with an error like: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: ```shell sudo pip uninstall -y certifi && @@ -450,7 +555,9 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing @@ -466,13 +573,15 @@ To trigger all tests run: make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. ### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: From 68cdf7de64f0e0e6193c99ad330dcd4e92e6b4f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Sep 2024 01:17:17 +0200 Subject: [PATCH 2227/2295] removed .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb deleted file mode 100644 index e69de29bb..000000000 From 2ff0c6b46c0cb830261565739e20e365dad717e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 18:40:41 +0100 Subject: [PATCH 2228/2295] updated azure-pipelines.yml --- azure-pipelines.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e26e7f760..b430c58dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,19 +22,26 @@ trigger: - master +variables: + # ubuntu version + os_version: '22.04' + pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' - # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-22.04' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' # unprivileged container without sudo, cannot install dependencies -#container: ubuntu:18.04 +#container: ubuntu:22.04 steps: + - script: cat /etc/*-release + displayName: OS Release + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' - script: env | sort - displayName: env + displayName: Environment # doesn't work in container due to unprivileged execution and lack of sudo #- script: sudo apt-get update && sudo apt-get install -y git make @@ -50,7 +57,7 @@ steps: # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( - script: | - sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c ' + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' set -ex cd /code setup/ci_bootstrap.sh From def50a98abb78cca6c938bb9fcca1ac660fde7a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2229/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7c73c3c4..399bc52b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 +Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 From 38c02005e8cd21b410901588698693ea6f26eebf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2230/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 092819070..81714c62c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 +Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 From 566d308bd84fa3a7ff327d7f05f67a83bb05c0ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2231/2295] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ace6d6400..575080b54 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a +Subproject commit 575080b548fec8b2420520e161973a861cbbce30 From fbca030c08ef4b3af3a131252dfbef8459bdd031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2232/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index b8d565217..da9d4c1ab 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit b8d565217a111514a2e061cec896f258712985fb +Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe From b77d999a26ca7f335eccc397881d3981536d8dc0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 2233/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 399bc52b2..4c661ca88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 +Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 From c6ab1cc4353d5a55bff3b41a775635537a9c5f7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 2234/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 81714c62c..666501ada 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 +Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 From 1981197b571c7f661bc912648878f6ef46cf37ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:27 +0100 Subject: [PATCH 2235/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index da9d4c1ab..6cee5eb76 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe +Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 From 3e40bd427b4780b74718a7dbdf395cf33484a814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:12 +0100 Subject: [PATCH 2236/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4c661ca88..3dd58df8c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 +Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 From 5b7e885adc73fada6c37497d18fabc7c1290c47f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 2237/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 666501ada..c90dd3e3a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 +Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 From bff7195cde897972a547360784d5a0f7129fd559 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 2238/2295] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6cee5eb76..2d12574ce 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 +Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 From c1a5672830bd8eb9628f505a305186cbbd65100c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:34:26 +0100 Subject: [PATCH 2239/2295] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f377f523f..80ff670d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ GitPython==2.1.15 #happybase==1.0.0 humanize==0.5.1 -impyla==0.16.0 +impyla==0.19.0 jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 From 5c900849a8d38f989a1343c3c36d54d7026adb03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 2240/2295] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3dd58df8c..adc18db20 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 +Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c From f373795d24110882a5e8a036484dfa0d5a423ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 2241/2295] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c90dd3e3a..eb10979e3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 +Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 From 2e739a43d79ff855a58c1d76725065c7d1257d34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 22:12:45 +0100 Subject: [PATCH 2242/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index adc18db20..6fc486cfb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c +Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 From 84e699a8bdae550158a4940234c98996666efbbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 11:52:18 +0100 Subject: [PATCH 2243/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6fc486cfb..176f383c9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 +Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 From b580060646cdba0f99af7e8bb557a53491d223a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 12:22:58 +0100 Subject: [PATCH 2244/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 176f383c9..34f300de0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 +Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 From c9f8ba3236600b8666053f3c9fc7bdbcfeff7dd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2024 00:07:20 +0100 Subject: [PATCH 2245/2295] updated .envrc --- .envrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.envrc b/.envrc index 7555ecba9..bafe8df0b 100644 --- a/.envrc +++ b/.envrc @@ -201,3 +201,5 @@ fi echo # read .env too #dotenv + +load_if_exists .envrc.local From 1b5dcec3b4c4ca4373615075b6db22c589a4d870 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2024 09:27:46 +0300 Subject: [PATCH 2246/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34f300de0..4f4be21ed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 +Subproject commit 4f4be21ed3006a756971134afa71c283388b111d From ee32cb23b8c4617015737f9c4938c852c0c03f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2024 18:35:23 +0300 Subject: [PATCH 2247/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e9c1c291..f615a7725 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,14 +30,14 @@ repos: hooks: - id: check-yaml # Common errors - - id: end-of-file-fixer + #- id: end-of-file-fixer # ruins .gitignore Icon\r - id: trailing-whitespace args: [--markdown-linebreak-ext=md] # Git style - id: check-added-large-files - id: check-merge-conflict - id: check-vcs-permalinks - - id: forbid-new-submodules + #- id: forbid-new-submodules # Cross platform - id: check-case-conflict - id: mixed-line-ending From f04002186cf34e64c7a7503d91144fa7521df597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 17:51:20 +0300 Subject: [PATCH 2248/2295] added markdown.yaml --- .github/workflows/markdown.yaml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/markdown.yaml diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..cb5395963 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master From 0e7f2891dca92120dc47009e1e496a871b8ac3cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 18:00:17 +0300 Subject: [PATCH 2249/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9fdde42d..6d11affff 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) From ac683274c27147af43de231a903b2e6945bd63b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:29:36 +0400 Subject: [PATCH 2250/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f4be21ed..956f47624 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f4be21ed3006a756971134afa71c283388b111d +Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c From b9a12fcc4a414016b44da2af0cd060f5b97dbe57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:32:27 +0400 Subject: [PATCH 2251/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 956f47624..cf080c2c5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c +Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a From 5206c0ff64ecd8f3b08d050859be1b6080011631 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:40 +0400 Subject: [PATCH 2252/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f615a7725..6ae7e968c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,10 +46,10 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black # Git secrets Leaks - repo: https://github.com/awslabs/git-secrets.git From ea2ed7d241ea06422fea231a2fcc513b17597ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:41 +0400 Subject: [PATCH 2253/2295] permitted space in section headers to allow for AWS [profile blah] sections --- validate_ini.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_ini.py b/validate_ini.py index 54a01cc63..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' From 67c680b35148ee03a249575ed54a0c047ead4144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:52:07 +0400 Subject: [PATCH 2254/2295] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ae7e968c..787fb79ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] + # rewrites python files with useless changes like changing single quotes to double quotes #- repo: https://github.com/psf/black # rev: 24.8.0 # hooks: From 510fd69c3f7c7ee0a98ad81ae26123c8f7fb1be9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 Nov 2024 04:06:10 +0400 Subject: [PATCH 2255/2295] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 92e3d2e9d..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo From 22fa4cff9849d3230cbbb48a1733c6403286510b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Dec 2024 00:03:50 +0700 Subject: [PATCH 2256/2295] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09026fc6a..ab6c6499d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ workflows: jobs: build: docker: - - image: cimg/base:2021.04 + - image: cimg/base:2024.12 resource_class: small steps: - checkout From 6b56bd27261b489177dece866bc49ca7fdd2364a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:21:59 +0700 Subject: [PATCH 2257/2295] added lint exception for on: truthy --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/json.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/markdown.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/xml.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a678d6592..dc5dfa86c 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -14,7 +14,7 @@ --- name: Alpine -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index fd88e54cb..a522558aa 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -14,7 +14,7 @@ --- name: Alpine 3 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 8f31b2672..209b3cef1 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 4dc3a09c3..6a0edd459 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -18,7 +18,7 @@ --- name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d56453263..7794a2914 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -14,7 +14,7 @@ --- name: Debian -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 735603e28..e8fec2ae1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -14,7 +14,7 @@ --- name: Debian 10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index de3b4b96f..3bebff0c9 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -14,7 +14,7 @@ --- name: Debian 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index d543710b1..3dbcdd303 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -14,7 +14,7 @@ --- name: Debian 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 769f90e49..7119d6852 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Alpine) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index b2b718ac0..24f8f21dd 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Debian) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 70c545db6..ad1a5ed8d 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Fedora) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index e69018c31..884917919 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Ubuntu) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0975be19d..3dfae11a4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -14,7 +14,7 @@ --- name: Fedora -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index c7ff16fd4..3c1aecadf 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -20,7 +20,7 @@ --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 94398083f..bedec7659 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 6a321f86e..351180e94 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -18,7 +18,7 @@ --- name: Grype -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index f68e309e5..83fa5ef83 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -20,7 +20,7 @@ --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 033f4331e..f7f7383fe 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -18,7 +18,7 @@ --- name: Kics -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ac6957e87..4959ef720 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -14,7 +14,7 @@ --- name: Mac -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 2d948e136..64ec035b6 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -14,7 +14,7 @@ --- name: Mac 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 7d2bf2dc6..5fa32d10c 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -14,7 +14,7 @@ --- name: Mac 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml index cb5395963..72e301ba0 100644 --- a/.github/workflows/markdown.yaml +++ b/.github/workflows/markdown.yaml @@ -18,7 +18,7 @@ --- name: Markdown -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index de6f79440..7d953dfd7 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -14,7 +14,7 @@ --- name: Python 3.10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 8e2d041ac..91658c8e9 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -14,7 +14,7 @@ --- name: Python 3.11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 58ad6c210..04c7146c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -14,7 +14,7 @@ --- name: Python 3.7 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f1c806dc2..37ad93654 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -14,7 +14,7 @@ --- name: Python 3.8 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 31c7d0ab8..9cf18e290 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -14,7 +14,7 @@ --- name: Python 3.9 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 43b3f5bd2..3ac4c40b9 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index f1e452f88..94fd43a22 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 753a6a495..ea4a967eb 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -20,7 +20,7 @@ --- name: ShellCheck -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 26f01fa94..c31ec991e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -20,7 +20,7 @@ --- name: Trivy -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 0d7fd952c..8edea5c2d 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5145177bb..ae5ec97ef 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 20.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index b9bf17a95..d5dfb94a8 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 22.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 53307dd73..2efa3ac6d 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -14,7 +14,7 @@ --- name: GitHub Actions Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 94a8c007a..468d875e3 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -20,7 +20,7 @@ --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index 8b07ed256..3f6265a6a 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -20,7 +20,7 @@ --- name: XML -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d252f67d2..5c3118c33 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -20,7 +20,7 @@ --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master From 7738761d337f64222906c6a623ed410dfb18fa44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:23:39 +0700 Subject: [PATCH 2258/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cf080c2c5..a97248860 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a +Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 From 406fce6a86160038a5ca122f54d0a401fab1e93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2025 20:35:59 +0700 Subject: [PATCH 2259/2295] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d11affff..22741ad41 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) From c9f6bce6e1972482bd39a67a793c6c241398933a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 14:00:14 +0700 Subject: [PATCH 2260/2295] changed concurrency lock to be workflow-ref format --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml.disabled | 2 +- .github/workflows/centos7.yaml.disabled | 2 +- .github/workflows/centos8.yaml.disabled | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml.disabled | 2 +- .github/workflows/debian_9.yaml.disabled | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/pypy2.yaml.disabled | 2 +- .github/workflows/pypy3.yaml.disabled | 2 +- .github/workflows/python2.7.yaml.disabled | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.6.yaml.disabled | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml.disabled | 2 +- .github/workflows/ubuntu_16.04.yaml.disabled | 2 +- .github/workflows/ubuntu_18.04.yaml.disabled | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index dc5dfa86c..9bed93efb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index a522558aa..f6c4f5ea9 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled index ef6aace7b..420b88392 100644 --- a/.github/workflows/centos.yaml.disabled +++ b/.github/workflows/centos.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled index a891a3dbb..ec764d288 100644 --- a/.github/workflows/centos7.yaml.disabled +++ b/.github/workflows/centos7.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled index d39fc2b0a..76c476664 100644 --- a/.github/workflows/centos8.yaml.disabled +++ b/.github/workflows/centos8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6a0edd459..168d4914c 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 7794a2914..d84cb16b1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e8fec2ae1..e857f280f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index 3bebff0c9..ad765f1ad 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index 3dbcdd303..dfbc4ec97 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1cb4e5431..f3062b6f8 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e987d7e46..1b0e00295 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled index 37640f59e..b41ead7d3 100644 --- a/.github/workflows/debian_8.yaml.disabled +++ b/.github/workflows/debian_8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled index 42e28702b..451348091 100644 --- a/.github/workflows/debian_9.yaml.disabled +++ b/.github/workflows/debian_9.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 3dfae11a4..5a4f9a9b4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 3c1aecadf..6b228ca33 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -34,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index bedec7659..c74fa2a0b 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -37,7 +37,7 @@ permissions: pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 351180e94..1b0fa15b4 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index f7f7383fe..c389f5df0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 4959ef720..438dd184e 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 64ec035b6..cc392b105 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 5fa32d10c..0bcd6e6fd 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled index 7dbc25c7f..01080bcc8 100644 --- a/.github/workflows/pypy2.yaml.disabled +++ b/.github/workflows/pypy2.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled index ac669813c..101b36751 100644 --- a/.github/workflows/pypy3.yaml.disabled +++ b/.github/workflows/pypy3.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled index ff3e51dd1..1813931e7 100644 --- a/.github/workflows/python2.7.yaml.disabled +++ b/.github/workflows/python2.7.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 7d953dfd7..8ec3008b9 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 91658c8e9..0f27b0076 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled index 84dce85e0..91bd3e673 100644 --- a/.github/workflows/python3.6.yaml.disabled +++ b/.github/workflows/python3.6.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 04c7146c7..73a0045c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 37ad93654..abcbc4c68 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 9cf18e290..1ce7eb238 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 3ac4c40b9..c2f614222 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 94fd43a22..04a3cf197 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index ea4a967eb..246087bd4 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index c31ec991e..95b2571ca 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -48,7 +48,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 8edea5c2d..fe863f875 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled index 395184d7b..1442d675d 100644 --- a/.github/workflows/ubuntu_14.04.yaml.disabled +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled index 0a55dd214..a9655835d 100644 --- a/.github/workflows/ubuntu_16.04.yaml.disabled +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled index d909f9529..4b510ba1e 100644 --- a/.github/workflows/ubuntu_18.04.yaml.disabled +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ae5ec97ef..47d0512ac 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index d5dfb94a8..ba8a549b1 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 2efa3ac6d..0037e4ba3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 468d875e3..ab6ab9638 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 5c3118c33..f2a4ca129 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -50,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: From 5dbc2ec8cd49f8990767878568ae893ba7b4dd5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 16:31:03 +0700 Subject: [PATCH 2261/2295] updated .cirrus.yml --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 6916e2577..471dc55ba 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,8 +23,8 @@ container: image: ubuntu:18.04 task: - # doesn't work properly - #skip: "!changesInclude('**/*.md')" + env: + TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init From 10d0ad24c89cededf549e38c49f209234d90825b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:05:48 +0700 Subject: [PATCH 2262/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 761ef0697..e2b78c511 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -23,8 +23,8 @@ version: v1.0 name: DevOps-Python-tools agent: machine: - type: e1-standard-2 - os_image: ubuntu1804 + type: e2-standard-2 + os_image: ubuntu2204 execution_time_limit: hours: 3 blocks: From 0adb1596606a13a7e3ee873bacd76953d4c6d39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:28:56 +0700 Subject: [PATCH 2263/2295] updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab6c6499d..b22b98dbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,8 +41,8 @@ jobs: resource_class: small steps: - checkout - - setup_remote_docker: - version: 20.10.11 + #- setup_remote_docker: + # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make From c10d63286ad086d0fb4e3b403f823a38aa7699a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:56:31 +0700 Subject: [PATCH 2264/2295] updated config.yml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b22b98dbb..312bf1020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,11 @@ jobs: - image: cimg/base:2024.12 resource_class: small steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout #- setup_remote_docker: # version: 20.10.11 From d8d48a9fc94d856d27c805b56c2287e727debedf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:04:54 +0700 Subject: [PATCH 2265/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e2b78c511..308278eee 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,9 +22,10 @@ version: v1.0 name: DevOps-Python-tools agent: + # https://docs.semaphoreci.com/reference/machine-types machine: - type: e2-standard-2 - os_image: ubuntu2204 + type: e1-standard-2 + os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: From 3204b856ac322c4c2e3c5b30bbf562484dae5fa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:14:01 +0700 Subject: [PATCH 2266/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 308278eee..e8de9f5e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,6 +44,9 @@ blocks: prologue: commands: - cache restore + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update + - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 2895d5f6ecc4012b3eded89bdb4990f5e41c3e92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:21:10 +0700 Subject: [PATCH 2267/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e8de9f5e5..fdb60732e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,9 +44,14 @@ blocks: prologue: commands: - cache restore - - export DEBIAN_FRONTEND=noninteractive - - sudo apt-get update - - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + - sudo rm -f /etc/ssh/sshd_config + #- export DEBIAN_FRONTEND=noninteractive + #- sudo apt-get update + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 99cf65b7fef1586302c8f0979c4cda90aac1ce43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:32:25 +0700 Subject: [PATCH 2268/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fdb60732e..5d5a7ab20 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -45,13 +45,20 @@ blocks: commands: - cache restore # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server - - sudo rm -f /etc/ssh/sshd_config - #- export DEBIAN_FRONTEND=noninteractive - #- sudo apt-get update + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections #- sudo dpkg --configure -a --force-confdef --force-confold - #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From f21e0aba680245a800caf478d4b248c8e3870e62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2025 02:07:53 +0700 Subject: [PATCH 2269/2295] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5d5a7ab20..59aa80836 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,7 +22,7 @@ version: v1.0 name: DevOps-Python-tools agent: - # https://docs.semaphoreci.com/reference/machine-types + # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 os_image: ubuntu2004 @@ -85,9 +85,10 @@ blocks: #- name: DEBUG # value: "1" agent: + # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 - os_image: macos-xcode12 + os_image: macos-xcode15 prologue: commands: - cache restore From 1ad7d99584197c2cc80e91d03e6dc335fa2606f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2025 21:38:06 +0700 Subject: [PATCH 2270/2295] updated .envrc --- .envrc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index bafe8df0b..e4488d51f 100644 --- a/.envrc +++ b/.envrc @@ -71,7 +71,16 @@ fi if [ -f .pre-commit-config.yaml ] && type -P pre-commit &>/dev/null && git rev-parse --is-inside-work-tree &>/dev/null; then - if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then echo echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." echo From 6b1938824a0bf6ef7a278c2f08061801dbae7ad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:33 +0800 Subject: [PATCH 2271/2295] updated bootstrap.sh --- setup/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 94b10159c..d7bedf92a 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -45,11 +45,11 @@ elif [ "$(uname -s)" = Linux ]; then if [ -n "${CI:-}" ]; then export DEBIAN_FRONTEND=noninteractive fi - opts="" + opts="-o DPkg::Lock::Timeout=1200" if [ -z "${PS1:-}" ]; then - opts="-qq" + opts="$opts -qq" fi - $sudo apt-get update $opts + $sudo apt-get update $opts $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then if grep -qi 'NAME=.*CentOS' /etc/*release; then From 2efb7e7dad0b6979452cc4d229e50e72e32bdf9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:34 +0800 Subject: [PATCH 2272/2295] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 0b52378ff..b01a77d04 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then retry $sudo apk update retry $sudo apk add --no-progress bash git make elif type apt-get >/dev/null 2>&1; then - retry $sudo apt-get update -q - retry $sudo apt-get install -qy git make + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make elif type yum >/dev/null 2>&1; then #retry $sudo yum makecache retry $sudo yum install -qy git make From b13623f40359ddd19ff96f6c2ae78a56375b3222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:52:41 +0800 Subject: [PATCH 2273/2295] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a435bc5a5..2a53008d4 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From 0bba44755a7b874dfc582aa7e15d59c7a94d0986 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:54:28 +0800 Subject: [PATCH 2274/2295] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2a53008d4..d4434ca84 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia From 72d01f4ef7dc4646eb3a35d665dce0936a445b65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:55:27 +0800 Subject: [PATCH 2275/2295] updated Makefile --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d4434ca84..1d98ac524 100644 --- a/Makefile +++ b/Makefile @@ -200,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib From feb48b7a21ea137bc18ca92344cdb8a5e75d98bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:56:01 +0800 Subject: [PATCH 2276/2295] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index c86449aac..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi From 55835218d3367e90d76d6a0614727ede7351e1f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:38:30 +0800 Subject: [PATCH 2277/2295] added plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 158 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 plot_uk_marriage_rates.py diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py new file mode 100755 index 000000000..6b1297417 --- /dev/null +++ b/plot_uk_marriage_rates.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Plots the Marriage rates in England & Wales using the latest download from the UK Government website: + + https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import platform +import subprocess +import sys +#import time +import traceback +import pandas as pd +import matplotlib.pyplot as plt +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +# pylint: disable=too-few-public-methods +class PlotUKMarriageRates(CLI): + + def __init__(self): + # Python 2.x + super(PlotUKMarriageRates, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = 0 + + # def add_options(self): + # super(PlotUKMarriageRates, self).add_options() + # + # def process_options(self): + # super(PlotUKMarriageRates, self).process_options() + + def run(self): + if not self.args: + self.usage("Provide path to datadownload.xlsx") + + file_path = self.args[0] + + if not os.path.isfile(file_path): + self.usage(f"Invalid argument provided, not a file: {file_path}") + + log.info(f"Loading file: {file_path}") + xls = pd.ExcelFile(file_path) + + log.info("Parsing xls") + # Read the relevant sheet and skip the metadata + df = xls.parse('Figure 2', skiprows=6) + + # Set proper column headers + df.columns = df.iloc[1] + df = df[2:] # Drop the header rows + + # Rename columns for clarity + df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women'] + + # Convert data types + df['Year'] = df['Year'].astype(int) + for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']: + df[col] = pd.to_numeric(df[col], errors='coerce') + + log.info("Plotting") + plt.figure(figsize=(10, 6)) + plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men') + plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women') + plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men') + plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women') + + plt.title('Marriage Rates Over Time (England & Wales)') + plt.xlabel('Year') + plt.ylabel('Marriage Rate per 1,000 People') + plt.legend() + plt.grid(True) + plt.tight_layout() + #plt.show() + # + # doesn't show anything because the script finishes before the GUI event loop + # has time to process and display the window + #plt.show(block=False) + # + # pylint: disable=line-too-long + # + # doesn't work even with this hack: + # + # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future + # + #import threading + #threading.Thread(target=plt.show).start() + #sleep_secs = 20 + #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up") + #time.sleep(sleep_secs) + + image_path = os.path.splitext(file_path)[0] + '.png' + log.info("Generating output image: {image_path}") + + #if os.path.exists(image_path): + # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}") + # #plt.close() + # return + + # doesn't solve blank png + #plt.gcf().canvas.draw() + + # results in blank png + #plt.savefig(image_path) + + fig = plt.gcf() # draw before saving + fig.canvas.draw() # force rendering + plt.savefig(image_path) + + plt.show() + + if platform.system() == "Darwin": + # fire and forget + # pylint: disable=subprocess-run-check + subprocess.run(['open', image_path]) + + #plt.close() + + +if __name__ == '__main__': + PlotUKMarriageRates().main() From 1a0723e81e31574595e06f5cb7f85038aaf033a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:39:20 +0800 Subject: [PATCH 2278/2295] updated requirements.txt --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index 80ff670d0..05d204c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,11 +36,17 @@ ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 # XXX: install broken on new M1 Mac Python 3.9 #numpy==1.16.5 +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 + # requires pg_config to build from source #psycopg2==2.8.4 # XXX: install broken on new M1 Mac Python 3.9 From 53def06b018590b505dc43379c0b27e99d5e1fd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:40:48 +0800 Subject: [PATCH 2279/2295] added uk_marriage_rates_2022.xslx --- tests/data/uk_marriage_rates_2022.xslx | Bin 0 -> 15337 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/uk_marriage_rates_2022.xslx diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 0000000000000000000000000000000000000000..54efc33218a9d57c8f26797118393a120865a840 GIT binary patch literal 15337 zcmeHugGM3^)Wj2nYxaun~hS!31!ngMxq{K!bpw0so^TVrT1YX6tOA z>S1r@q(|>=V_mBzBhw>@+z$5?TBTu8PaP(yg2W$9UA*oxDxY3>Z~V68xWnzsXOgMV zy36?30q>Yw^Y|{n*M&_4)U0*nV znNfCN*;>ks*Vqzsm~ER^Ruk@fCs!U8Jj_w)7yG08ovGzSZWeb4J<}US99$5``p6+h z6Hf>d_7GuufUKbd&s1<}JWUOwwQMhGeS1Kz&zHs7ypyagNpX?09Ml!6fmOiQ;p^Bp zmBg~cmqbhTZyJi7hi?>CMsAuak*Gws_fV}X*J~7PTo{;<$(X;-ey)qmxF1W-#ivE^ z)H(6|fWAET1OF?-5T??uuHw%0Sr&_(1QmE?M1~7;C5!XG3b|<&LXe~rT;u7h@tpJ& z+$2E(R6ojoFU;@wUYXIh^7J+wG#vV&)2eeIT>&!Y*@ydO4vPhtX2K%xDZy?1IgsjMtc9-IcAxpRBO30d1(Am)VTx11{O zz026jjPyy`K|upZG1FLC;no(Jo4gG>`>LXLQw?v|8lis4+7PU)!6@CX_Tk`wVOs z9y7~nofbe;`M5u1IAsm!`Yy27oqaVO5aVG#3%0>2qZ1ukn*H|Rl9k`d$}P4OKbL1_ zauMvZ^RYDBuX!2f_6KEe0w{`NzqBT%q z68cV;%yQ9fBsVomoKq{E;n% zUSP$%fki%#TDrI#Pg0U@wWIYNBRcMtk?_@F!7f`Fx36Jnk&8>>MLDZmbRT)s$l50U%zGZeT0vMybI9Xkd6rBcWqEr z6K6QAzLls=QKsV3s~!VEV_d8bHsSdO(t`7Soi@I`B6ahar0@ey;kVj45IFi(i;r12 z0DnzPPH68NR5jr;&30ii}w`@+B52?HQDIrqEe)HIe38&FDa@R$8e601?t-;JO zazQqnTn7Az96g(S3~G>5xLov{)*RPxTcxmACIgps#|8PRQKR!;$EV05z_)jLMS91; z(BsJj^Jytw5$6(eZ*&fNN?353_%GUXEOudX8C=F=+gl^lh@{GihbOjQ-asVT7DPtlxjK#|;=vVK378N9#pZe^tW4qL?~~lvEOJMK z@<{j=upYek{dSU;hI&?aYV#GpHcP#1C$!=aG4c9+mh~o7I3L&tvH>m<&RV5-cZenn zen+1Sd1ekj($Z-I-}3_4vJD&Y*Q#yPPz3#P?K}4iTicEt)u%Ji_hL1w6kq)A*T6c2 zuGn~3I6HV)?9W}ev$L?h5=IG6E}R0lEaLlj2L@`i9`1SXkE?<&V<787LJkj!R1S#} zN<02_O|KQarNjZ!pD6?g2+kjC`VR*D>&pJchOLQmRy~aHA*X?l&|CL;WGl_g2S1ph z+HA3pEb#z={jOEBHn-O-*fJ`V3*@v9Fj`2#V49?-8w@Qq=EgX+tz1ET1h_6f^tI08LU1w9ehFDjN(!LvXeJ5O4D=!0^ zTbh$_KtvP78a=;Wgu}}-ZF({e8 z^?@TAj5KuQDn@3pG@qhWAYe%nS&Y#E6rI72UesTZ3wyB{*4p-P&~ZB9KLsy zBtDLS1IxVz9?;?u4LG9gRm#j2%=w799Y zEa~&~L(pKUEOC{Uw0l8+l2Vs0j8X^dU%&0}>t|00&9|J;$D(o3F~kwa3Qs;%u1^OO z5GW5I7N4uGH;u9t^EU|ULz<2ud;@J$UyspEOjKLnS?9Epf;?iCn}Xm%p`G%%R_SY3 z=>qZ+c9%4RGD3JhCFzGDjBFSaCnOP#q;2EeTlw(0Od@gCuR@RY$(el9=pE3XQWj2{ z-&@Us8q?wK>+_*zHh4{obo*xZvc zlSCRwHaxBc=Tb2lGu$Jz=51ali9%zldk2%B2AN@McIcY7CYbsA?bh^Jb16s&$?lF2 z=^@4&VihV?4P#!gKCxy@bKV7hN>J`byw)QoEdd+f!%}8Sm45)eBkt>J#~~e=;<*a60WXaq_t zRqJiF@AqI5bJT%v2{G(8`|G-skZ)856sjKWZ3!h;geYN{YvGukfsnS>$d_h+dUfiH z(*4DQxVMy1YLhb8j&j;hI_#|(Y7v#s5*qb+7V@|>&W=Y`fz;EQW%%h8!!Hy0^%AMQ zWQ$J&|CmHg$cSxddruiY!*_%fULx(oy;~fwfosX=MD_*YEcBZab^p`{DWN6#C&X`*!~ruw?WAJ2yW63F+p!vf&JHJHIq?!O?p{ zw?lmee`cvz2?c3|aG*7T0p~f9^Yd@+>Kr=uLm>(_N==5q`fqvAnX@^DbpZ?#K1zxg6k#r6mT>(wCQtpg~kFo#R3A>XUa+c=n&SA&j* z6zi!FLoC`^F>ngqb5VDf*wdqD`MVxjc+om;%d`TB)^t2wEB$9M?#&6{I>Y&3nEPAw z^qxkW3ruSXB1Q*u#LI(7qjj0lS)mu^+nG?j7v{Sc#hrb6j-GU&POCx;!_ymz?v>y? z@A>*@zB5-vU@@HxJ`%icES8sNP%s?h#qY=*$~`${r!~3nVZg_yOMdN;rL-gVhLN2@ zm_@Vly)AX?xc6`aLadV?abV_X3qHcFbp`Wg*)TP8=Aqgi`8@sy*JYLgIC`Q{>I~Tg z3^$JUauiY;*^9(^*-P$|9cJ!fgj6mb#r!#?j}5h(r~-WMc=RZgvsYNIaCDt)n&PXdp~g5 zFM#HO#rA?@K=G;B_bxA+bY4MF(zoiaT16NAAgpmzV0^OfxjFhSFQ0T?Nl@~s>aIy8 z;}RT2DcvV4H=?Z~yM1qW@DeTd>gtPaY+m$Q6i|kLe4$`LTQbU{%@db6(sRxseTvgAkK>9QqM94Ws@pMo zHnhG6GRN(biR2Z8o?BPs!>yZzWATGV93MK;CLEGOx?jghh{)zPI8I8Oz%#!bVzA$P z-^^M0fP1YU>0wI`NXnH8dbdtzWyvn{LU2Qbv)U`OH)wiUVg`h=a-)Q#S6FER%_kvc zEuGni^hCRl2=XTalK`<+)0~dB=f`1OX8`0yoIqzbjh$oh?H05`vp2gZMgAjoy9r8z z#&l(`kcKhimSrOD3BXXA8|CE12hGvivs1 z_OM1DoX252XEP|43US_)hGAU6mZN2o{JsQX*=YlHV%~J$Ld0g}Kx~bkByO&m8m(tn z_HtVhz4cQRtWo}S-<2%q6`*Lyp{rwYX;b(q+Xv`LCsNzFl{w5J(M1bth`QlUJ5^uYIRp6MSvN zaZoM~7_V_}k^w!m%-xwpBqp)FIDP(-Mei%2f$qD7ukKi^@UUrQOVP+%4z!C7gAEG3 zjjoJqKfupR4?-JSp_1SC-?TF#H{4!Z-4$4I11IIQDqaLI?nf*!q4khQPGVbgs_{cg z!*UW!@Rq2niQr4eemGkqfoU(H+NSd{J<*u%<@+NC4 zCJn`}T1})8lOEqtBXzx9==iDC{D#)N5hJPJDk*trw;x(dHXBQ&?<>b^Il0kC>M{Fj z;pY41lQA(jjf$4{83sk^4@)2c4Ci9F4qn$h&eyjOG`Rx5d7d>&2r)nqUaWHJ2S@-@ z|2r&oTiwP@htQ zrYtD@C~U`66trjz}wI4N1Z6(kcQQOsW1-onv7$Sh_1I-=MN!3=x@zZJF4 zO(vA$TB642g9(Xi>@?@}UwacdAlXftjWK|SAGn%=K~tz)jzt=OqaR?b=;gyqOnQWn zB<<(*xm96q?$apdaLJ)a9Hr3@(Dii;<-^46A!_r*VHv7Dyr|hTk^+U z%mxOGN%c3P04OaH+ni_SCRsyo%}J;8{VvNXBA7sJuR;UxV%>qu!$<9o`l(N#?^pwp zit4{{A12wx8X+j&&E~dfoT`mrtWFr{)imLk#01ahsQOif5W+XFi@{Y2QSs~cgl3w5 zdV4sZgKnVHGcR+juh{UzN>NnGu`z@`xX#k8Y4Npb_g*yGDtw?&TRhdcQY=F0k+O=c zCJ6^XJMx5?ALKolGMUuoP#?U)1k9Jl2uiN?-X-#B7*x9u)HjBWs#FNlYB!bq{!c}u zAL<#I3zGBc0uiTA$hh*y?XmVybh#0c?yq>=AUxaq>`Tf|t?J@Y$R7Wji3r((~ zaxxoZ@cj6QxoXhy^!?zBV}FL?th=}4;dbO>$MYHYW!2{LYle28^Yi@#%r&3OU4NOitfK82KaKpMeTw_A(0h#tWkCxfY*{41($F5FK>8R2_FO;c50m zw(@pZEi@CJHIYT9R*F|cE}KNUV!3gzm!yq+|8AcpyOMCx6y6d(5odKmFI@(5y~~oL zv8lWeRb{q=@BuZvh_h=-rK4IT62G)XBn&Xm$Gj<975?J{xiTU}L zq}z-rXz_e>)uyti*lL!~+mzG@*bheEu+M=0k$3b;N6K8HOlqb zL_I%HUEG<_2FdJjhhE|cb-==JA>I?{^|HsyIPOg~MPRYS@wtABQsi}7l#L(k^?id# zEEeY$D23@Av%n+DCxWN=g=x9rRekrPIY}erwq;r(McAF$7=CqYWNI5BrgK15gxMl* z{kOW&a26i!i_b}mEm$hfk^NX+4g5};6Gki}yG@fCB!)@&N77|9gJWu!8HS){@g-Js zRE{5EXw0t95^bv9+BpQcV^D^ptQFkih^|QMaRM?_Ju{;b<<{B20EF8S(^T`2dyMfd z9a4u;Fj{XvdSaY->)F%z+SLpg8m#giw?kliTVfMkHC&A%D_I5b8il~X!eg!kguQ;! z70oQwwmwNr|CY{<1dcAQ-iDU29o07;{)OVfZJ59f)CmeQM(6B4kvpBzY**C3=G8SJ zt(E7ytmRJo_%f$C4>c$K*_-G467y+&{z|hktJDka{>!UNAFP#CMUlIcA zBwn5kjmCRqe(kL^K^nqz2V`fSL3idZQj3rkETt@YLjMhw3j3#$zBT+ZvIV^EHo~Zw zHgN7;4wM2-O$Hk1=ZkvX~@8(-SOmQVflX6EWy5^;rW_7cL zSA3mRpdrLkSV-&2J*4MNDFMG~Q&{03oE>(8vex^-j5Cjz{SkuLiyFyq%M-}9lSMro;*g%$>0VTS$2#zmZy@2rAC zU@-Hmwu>O`p>*cW-SBejtU0usKd+~~ALl3KJw;)=kEE1$Q@DeP?1mA`30hGwYpq-q zbkDw2*3c``-{=x?j5?521C?`8>Pv%NvD8N%t8@=ssFBtUYss`m49Xkgobt-Hi7A8cHkC_0CXIH+pm+ehqam0&$K{++L|3OE#RA7``ope zi33wb23Ar;rCeB^HY2}S3jU^ghj1_@%z5#-oh3G?c)Gccp^N{X-_LDrg}mtmLc*&A zx5)J1Mk6b*m{xUYo86_I4{)+96h-`nOjjz68+ws*eqjJf$^x)747YF1}?CN2I;wS2%<=Y=sXOv2dvD)hdN_b_>WwRJj_N@zXo zkt|Mo)p%qd0FUa_a_o4;uD+76Gq=@!l=KSEgsedrB_$$gT0GC0b^jK z8k2ngz)mL6t#6Ow^=T~(-B|AW>+r5=xV0F)Y#{8cG`M4y>^sgThAym7L>$#7HS+Bt z@JWZ{^+_%nYH|jqGGdvux3v^_AYfi?in!Upmy4kah2nCgq1ej3Mu{YZGC2)~ zE`FNsg7{obO~FQ{iKH8iL$)os%rO|S*lO+~pIQ_s+@f%#33ZAid5 zMJ^zzo1VDefomNBH$p%Rw~C^QMRM-BC$EBg!@xa8>cb&QI*{;)SZcb6XuB` zvt`EH&>3*^h0MMq+`eyTQzPD zn|G=306^$SNDtABNtH=h%v+rUS$6P8{nt5sfOZ(;(>LdQ9N8Z2xc#OlV_RC=J8^_% zTlOxPfXN`QAN@*uMr(v2CFF#32xy5c9y15U6Em-nfR51}6)q5f?#^xtwf zJOP)92QUHov%>asF}yQ!G&5CkcC@g4`)f@^s>)d{2_m;)-VuiI@g#4HOkqjoChjCd z;DjnlBbj(u(;bC2*Ek;et!Z&t;8IQC1=U{M?zCwoX+T>UD)`sAB~kA*fjLI4y=y3) z={dMNN&t)AV}IA0FO45QtM+&%GK44Oh_6u+Rjp%=%|WNN-E=w_{`N4ik8+@Dq!BI% z*ET$lZGmKboF!tB>YewgaUne%WZQIy=ba-nI%WyI);wUsbO@h)O1FeDnD6tQ_UWBq zm8NiNU-MVxJ!Jh#ZB&T}cqt^0zi&Mp_IChI3 zh55M_OX>4$X71nhcC{q9O}O7P!bDyOrubbKJx@3utO=uW-f0G!9Z=Q*Ih6bh;q zECRxNJEmbXQSfV}cFu#4fzgvEz4G}(-Yz!%X0(W{s?#7N>lP`;ugrB& zOxK5QMs%@j@Fx*>uY|c^co7v2l18;%aO1m0sHKmw*XWO4HF##70D%2CfIa=5_cN|J zVjS&EGg$Dp3Ff=1?_P^^9YWQd2g~TM=y+p|rs><$z~$wo!v|tDU&B=gE0H~7_4xKK zR7b!19LvA6AQkSxICL9)D>UCSU}cS^hMqY=o>r>ymK6zxOEYn$IOCV4&{ zB*hhL$e$<4H7Is`{@z&V(CsSA$_Ejm$mZ+F#nr2S8-YNIyfy=%=6k?K@y7_*1DA@k znWKuCv-8h2!|>uW6TPv8Ev&hu*`SG}k)k1`(I9mEiQtz=;`g~>#AuZI-GVTTUl^-a zjn<8jvqlCx2RnxtBbI_lAqR~OCyY$Nrisw-uw`M=M4Q24>4-q`jI2#ny!CX!u^@;X zfgSkEEd5y_$Azwan**F7L14KM1$c+4or$8OoxKx-iJha_&uuO;Ue0dmXEg!*ju1Zc zaBz4Yiqyg1b?-AM6l-6|3TwDr-Ece+lgs0P;IA%0c5WV`nbafR)wReGC{Y<~+I zT0(%?dg6Tl@!XMxSGJ##mhgUU?P^zAUF+2Fm zKt#ht>r&7T()@&kb~l*%h5Z{;+Wy1|mrf}$XUSRucCg&EY?{^@)Fs1B%xI5^fTgW& z0}W)EwuM@r5ZvsC?5rQs_XZt88|wrJu*EQ_K4k>(%8NCirIj^N#8q_@BGDgguPZ0m z`42=h;35QZ;y=70dfZb4bPHM1pxN$4d=6>1c~<*wXQaW=oxX2tR_yhq_Mm}B7L+^R zC#lCZChoMRrE3R{Tz-jydoKob!PN$K* z{jXX^tenEH+K}W!DCIR9wj^5Gl``{~?B zK#bjbl)azdUE@Z&1&&{-s6H%oQiLI^8VN@H@q~#dv?9XvC6s;wZh7}r^RzUmV@2!E1)(1W~4uDeNw1?@c`(`eW3g!n_}YPa*<#%_rn`H4U0S!jZPDgFUWMTqj3x8bjvlygWyedX8F#T8ezE zV=Zo`g;q=bixJiwm>lh4^gzp&lN-|g)rt`zR!pLU0PK7bOTAoR2q%f}O%*%ZMil+1 zn}%Jr<_0A?EBf^~Q(F&Dvji0HLp{5kNz8e6;Ya>_KE3oM7z!_B6}zPN$40&Mt1j2| zN*lY3!^6Iv^Yi}dPh-|C%eEB_W7f8zFEUj`EN@Uk@yu?`AWX5d=0+cLWgTyf3|XJh4h6aY=zcTiEss%eg0tQbTUn)< z&O^MJWaa>P&A)n|BF1~(izVSjbIwbsVr9I9IhO@18Y`$igp!)ahm%KXZpeqm50JhU zhuLWf&slEuyqI1Br=YP(B1ZPg5$+EffawVt=V;cH;ffGLInt{NDv8Nb0Pik5eDkgT zjJ(Yr&#kEna~h7huI<+OqzUb9ePbhQRIGw?e~@>H-{csCms3wxR#v~lv_mgJ!71Kh z%#s54F>@%s`?K5D#}kbuXxB=)1AXJA!m+SsCp%WBW5By`WNbNIW(gJM3q0s^X0()w z(a)hab{p%H@!aoNNeKvYZy};6wdi*vQaRFdwFv0_2qk2(j8Ka854E%!)6d(I#{m|C z00%XwvQg#18c3o~Tki?T;J2y7{m9;9-|m(~A8oJXA@flJZp~x%goR>1kIpb2l*^?u zyYci=-mYM_7;p)K6l4&kNvay;eq$}H5Qfv5)D4}f^*)Vspc+97m3vCFc+DzTd3*|F$o=zpYa;Yx zFuG-yaH(%7J=>m0@0HF$E*Z%CySSWAXnPV!+Xp!!kt;}&%nHa+6Qft0D#*$R_)I6E zTE2T?OsG8J@oistWA#j#Jj4N zvWgMSjF1{`rBa79p%{epiBTcBw63WOs|=ZNqu2s+!4{AJt3FRUi+Ze@TgLWi?FkA$ zTODB@H?mQVy}?g3|HC=1M~rK@#@$0516K`Upr+|TUWVW49WI!5#eSq?7;kc>GnI>g zM8FOe)a`Y62$Jzz-aI1z8-*Pwz2GDRX`BzRN?N<_DEn$io@{LW&zI{R`bgS1wGW(J zjAL0Pu;tvY{KV)Xo2@CThhnA6R(Q<8VK`&lnaU1H0o1PETrsIQ1O@f^wrljap;>Das{^w*}>IdYuTqXn5 ztHC>;Sv!*;XE<)tQQ$=>#B0jrHRE)gx8=;=M0thUct$>;8x>HEss@nRu7gXB6fdf9 zTWhat+CIL=@22FeJ4!WEpbZ$Xe+TDS@bUFRh=GYmVX38v%lFtN$^;Zo=f-W2<&vGTAr997ILl`33T+c_N|SB<_@ua$ z3g-I-E8f|tpw2An;>f=;%;d}J8_t&)6Yj~dp)8yyFKsxP z`kUa;!qpBNYAvK9)p__zU%l?UlPst)ZxFDq5Am#`#y;HWMm_?zV_-W0w$qE?Q}ROQ zMoFptr|YawmZ}WsmBwneor=Zru&QcY&l#WV4rheHRrNPwU$Y}d*f}gMO~O&}IML5= zUv<2yd;;ck{^=UnLL80t6&X(a}laKq#6)qaXc^29pU+1B<@XlMUOGc$I zZ=pq%RhnNx5is#v6$W_aG#Cvj6hJ^{wMBg%bL;D2q~aBNv-9h> znLe=z(Fw}CTMhJ&9In&n`~owCcDGZlO0#6Vjcs;%B6z0KyhtaO4%d6pCI-#Qg^G2e zi{9vEdf3ax>f9g*aAvC6D#dxIqtHV{OhtnB^Yz!UL-xWVH)_k6PSZcY%MTCc13z-x zEfDCBfBLB4^-&XnYr;*f`*{vcg6iGK4qqpWx=D#bw>lut_K4!uysY(G(+rCi!=sJU zhAkeTI{#Fdz{aY+7I@`k>Y{1FBud^q^^uwgs}J|9#vERK-X6dLreknfvfp2*^D~9lA$%wZ}eJ zR2PxEkxGe1v>vWd?MbEF<#kDNJ|$*b(ejIRwr2S@CM_mQ3v&11N;9gq@%|EH6dcH&Gr0C}j=^@ZSHwe}cxlK7cX7L#?;0^CT{3j=3Yp?YnDX zw1zjX*Eut@^jWB+QQlo)DA3RzF}=EdDp(QdE;~tCP+^@=WuuIY_9T=57uY?W#XygU zYRja`^QT}wY60Z&ZCDr5o^7L*=83X1h8JYc9le21rjf=PG0nj%OJt7W)^e*OJ#f*p zbAGZByf!Kku_Ge9gPOTbl|FxFM+kJg5j(+P9416rT+yV|?e_kf*f_`@LkZPt=-!Z4 zvJBU3bvfX-^0x*4rxoZXQ;_uhFIMo&;Q!AG{$lt*E3gCx$588Dxu=HC1{qh3(&5uG z*xT&laBVmeQnDr^n$i|)zD1ufpm?N(*BIDBy;5coSiC+!v8y4#=I5^tOm^P>xU2^6 zGd8h%`{CWXKYXZbo29RPy>hbqdAOPq32ut$l)jZK8q~;Z@<=mgMMQP!REf;8mPM4y z`U(LTU$z_{7PWcYLA?%87tezDhYq=(b$o&$I?((Fl-!kW;SJQCQ`ztc>x(esPPmY_ zRys*uLJd*A_mzJ?KJaJ2+ZJR@8Vgi#3E2M0^DIAk-a48G5y*Iul>t);bbAwJ1g>PQE_kyN_|oT@U1)D{cncRwK8{B0P==Z-%Xxt~ zBv8=ML4MS7pP6Iwmu0{2C8p+e{a$BL*QKd9g>@9DYQV(T!ZSO=rCT#f935H0bgW0Jdtrz zqI4i{M=xdlo9oFZmcn|KkjruVvKKQvZ)-$ z_T|G5#3DVkT$u};WbTQmbGwnW}p1 z&6OFx-`pP8&8k6OM+(0l&Z==33EMA#=;t+b^dwVXq&2-5tJc$X>=>6w=ru)4?L+Jji0Ak?M1c5kEY9mqj)3l2HW8c zs8x{n0m*=hu&ocmk_`b7dFaj~SacpJ7@>*FZp!EB)Cxn7skfMavz0$N&+7s#|9|7W zU&iu(oCh4gKYxC6p3yJPbNN*uezzU~^cgLt{njlD>CaG^OH0VutHQS1=8iBTlW~Vr z4or(s%V)m0XO`dB6I|BY7`QaC88))aa$-|%&G>WcIm-E3PxC^D+HH@aBs`$yuA~NU zYUr-PnN@tDBj^)Ej+~#9Q8|U=`*C;Du>nkh{y`a_V6eYRQZPWR{`t_?pQ`@x{x6S! zDa!mE;N_D>zX84i*XCcIJ9;Vj^0}Meg4e*P@5as1Lw3pp@Sv&s?AOH-ofB?TM z=`TfJRvLec%HsYZ`m*l$viC1*Prnho@n4R?zgD7N0=_Ke{08(T_yzc)kn^vFotJvP ztiAjOB_aHmO3X{~m%0Am;%7vE5&uu-|E2WHWcY7sM3R>g;jhs5CBVy&`8NO+^-F-? z0rX3hm!ary6ik|zD8D?`mjEwAyWapXv@ZdEb>qKL{%Y=j51W2tfPkF;4dvg0sGmLh zE7sqg@4sT*(EqhZzp;M1;xDy(>B0R*duIGA*3Yo+*XH&T;H3xk8=#ZvCBU!Y`B#*e zzR7QtGUmUS+8@E@Pfz9LnEZWn{w)pya>Vl2Zu}mT-&?h!3?z^MK|nBozw5wbvKMSW GKm8Y$>}z%a literal 0 HcmV?d00001 From add60aa8705f2481287cf0fe9b77c0aa7b4e78fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:42:05 +0800 Subject: [PATCH 2280/2295] updated plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py index 6b1297417..a38332443 100755 --- a/plot_uk_marriage_rates.py +++ b/plot_uk_marriage_rates.py @@ -68,10 +68,10 @@ def __init__(self): # super(PlotUKMarriageRates, self).process_options() def run(self): - if not self.args: - self.usage("Provide path to datadownload.xlsx") - - file_path = self.args[0] + file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx" + if self.args: + file_path = self.args[0] + #self.usage("Provide path to datadownload.xlsx") if not os.path.isfile(file_path): self.usage(f"Invalid argument provided, not a file: {file_path}") From 76d07c239817b67971c1e96aa9246abaf6e7b133 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Nov 2025 23:18:46 +0200 Subject: [PATCH 2281/2295] updated README.md --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 22741ad41..40045f8f9 100644 --- a/README.md +++ b/README.md @@ -637,24 +637,35 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -### DBA - SQL +### Databases - DBA - SQL [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) -### Misc +### Desktop + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) + +### Spotify [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) @@ -664,7 +675,4 @@ The rest of my original source repos are Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). - -![](https://hit.yhype.me/github/profile?user_id=2211051) - From 58f3505733d4fedd2abd8baa6bc69a64cde89580 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 04:09:59 -0500 Subject: [PATCH 2282/2295] updated README.md --- README.md | 55 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 40045f8f9..e8cfc70d7 100644 --- a/README.md +++ b/README.md @@ -601,8 +601,8 @@ Hortonworks guy Jonas Straub: ### Knowledge -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) ### Containerization -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) ### CI/CD -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) ### Databases - DBA - SQL -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) ### Monitoring -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) ### Desktop -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) ### Spotify -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) The rest of my original source repos are [here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). -Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. From ba4689ad10b7936ce3b07791671bb0c2890e2b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2283/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eb10979e3..d65ec80e0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 +Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 From 572228d476a5cbbd8dd6dcf34217b4d5c3e82738 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2284/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 575080b54..fa8985e60 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 575080b548fec8b2420520e161973a861cbbce30 +Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 From 8c8d338d2cc4b7ef9282ebe9655a38eb31a4963e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2285/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d12574ce..344a446fc 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 +Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b From 351e28284fb9a24d1d187c213c5f090ed99dfce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 2286/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a97248860..22c6dfefe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 +Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db From 824267aa84016f4cca4b4d6e3b971ef6a91d5a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 2287/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d65ec80e0..9efc29299 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 +Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d From 58521d67c5239a3516f9fdb08a35734f46e6ddac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2288/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 22c6dfefe..8652100df 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db +Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 From 40f0b93ba154d7da4c0b2c5f565fe89df6bb9f39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2289/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9efc29299..61ee5978a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d +Subproject commit 61ee5978a396f733033af23860014e91969f07e0 From f3cb837c9c2dc10589deaf92c912bccfde093c45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2290/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 344a446fc..620b30292 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b +Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b From 555239dc9cf306ddca0f20cfba69f84de122f431 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 04:56:20 -0300 Subject: [PATCH 2291/2295] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8cfc70d7..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -664,6 +664,7 @@ Does nothing: [![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) [![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) ### Spotify From a70079597d44a1b0df2cb585dcc3ff96131b73f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 2292/2295] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8652100df..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 From 511b9e6f20cbc66a10f6d4ed88a45311552f5916 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 2293/2295] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 61ee5978a..0d272f4cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 61ee5978a396f733033af23860014e91969f07e0 +Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd From 874083a6f3e5ce387d1496b7509c59b30f05aebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 2294/2295] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fa8985e60..30d80bbd3 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d From 32ffee202073282e7b7006102e692c453e98b01c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 2295/2295] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 620b30292..f59532dd6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec