From 98b85628de25cf776dfa04b40575e8959c019f40 Mon Sep 17 00:00:00 2001 From: Federico Luzzi Date: Tue, 12 Sep 2023 12:33:33 -0300 Subject: [PATCH] Add bulk edit with regex and export/import folder --- README.md | 2 + connpy/__init__.py | 2 + connpy/_version.py | 2 +- connpy/configfile.py | 33 ++++---- connpy/connapp.py | 120 ++++++++++++++++++++++++---- docs/connpy/index.html | 172 +++++++++++++++++++++++++++++++++-------- 6 files changed, 266 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 33142f3..15a0167 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,8 @@ positional arguments: copy (cp) Copy node list (ls) List profiles, nodes or folders bulk Add nodes in bulk + export Export connection folder to Yaml file + import Import connection folder to config from Yaml file run Run scripts or commands on nodes config Manage app config api Start and stop connpy api diff --git a/connpy/__init__.py b/connpy/__init__.py index 6182528..8cdd75c 100644 --- a/connpy/__init__.py +++ b/connpy/__init__.py @@ -39,6 +39,8 @@ Commands: copy (cp) Copy node list (ls) List profiles, nodes or folders bulk Add nodes in bulk + export Export connection folder to Yaml file + import Import connection folder to config from Yaml file run Run scripts or commands on nodes config Manage app config api Start and stop connpy api diff --git a/connpy/_version.py b/connpy/_version.py index e1d7395..1d03095 100644 --- a/connpy/_version.py +++ b/connpy/_version.py @@ -1,2 +1,2 @@ -__version__ = "3.3.1" +__version__ = "3.4.0" diff --git a/connpy/configfile.py b/connpy/configfile.py index 0fe42a7..e9165ff 100755 --- a/connpy/configfile.py +++ b/connpy/configfile.py @@ -203,7 +203,7 @@ class configfile: ### Parameters: - - uniques (str/list): Regex string name that will match hostnames + - uniques (str/list): String name that will match hostnames from the connection manager. It can be a list of strings. @@ -214,6 +214,8 @@ class configfile: ''' nodes = {} + if isinstance(uniques, str): + uniques = [uniques] for i in uniques: if isinstance(i, dict): name = list(i.keys())[0] @@ -303,7 +305,7 @@ class configfile: raise ValueError("filter must be a string or a list of strings") return nodes - def _getallnodesfull(self, filter = None): + def _getallnodesfull(self, filter = None, extract = True): #get all nodes on configfile with all their attributes. nodes = {} layer1 = {k:v for k,v in self.connections.items() if isinstance(v, dict) and v["type"] == "connection"} @@ -323,19 +325,20 @@ class configfile: nodes = {k: v for k, v in nodes.items() if any(re.search(pattern, k) for pattern in filter)} else: raise ValueError("filter must be a string or a list of strings") - for node, keys in nodes.items(): - for key, value in keys.items(): - profile = re.search("^@(.*)", str(value)) - if profile: - try: - nodes[node][key] = self.profiles[profile.group(1)][key] - except: - nodes[node][key] = "" - elif value == '' and key == "protocol": - try: - nodes[node][key] = config.profiles["default"][key] - except: - nodes[node][key] = "ssh" + if extract: + for node, keys in nodes.items(): + for key, value in keys.items(): + profile = re.search("^@(.*)", str(value)) + if profile: + try: + nodes[node][key] = self.profiles[profile.group(1)][key] + except: + nodes[node][key] = "" + elif value == '' and key == "protocol": + try: + nodes[node][key] = config.profiles["default"][key] + except: + nodes[node][key] = "ssh" return nodes diff --git a/connpy/connapp.py b/connpy/connapp.py index 634822a..d1ecc14 100755 --- a/connpy/connapp.py +++ b/connpy/connapp.py @@ -13,6 +13,9 @@ from ._version import __version__ from .api import start_api,stop_api,debug_api from .ai import ai import yaml +class NoAliasDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True import ast from rich import print as mdprint from rich.markdown import Markdown @@ -102,6 +105,14 @@ class connapp: bulkparser = subparsers.add_parser("bulk", help="Add nodes in bulk") bulkparser.add_argument("bulk", const="bulk", nargs=0, action=self._store_type, help="Add nodes in bulk") bulkparser.set_defaults(func=self._func_others) + # EXPORTPARSER + exportparser = subparsers.add_parser("export", help="Export connection folder to Yaml file") + exportparser.add_argument("export", nargs=2, action=self._store_type, help="export [@subfolder]@folder file.yaml", type=self._type_node) + exportparser.set_defaults(func=self._func_export) + # IMPORTPARSER + importparser = subparsers.add_parser("import", help="Import connection folder to config from Yaml file") + importparser.add_argument("import", nargs=1, action=self._store_type, help="import file.yaml", type=self._type_node) + importparser.set_defaults(func=self._func_import) # AIPARSER aiparser = subparsers.add_parser("ai", help="Make request to an AI") aiparser.add_argument("ask", nargs='*', help="Ask connpy AI something") @@ -135,7 +146,7 @@ class connapp: configcrud.add_argument("--openai-model", dest="model", nargs=1, action=self._store_type, help="Set openai model", metavar="MODEL") configparser.set_defaults(func=self._func_others) #Manage sys arguments - commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai"] + commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai", "export", "import"] profilecmds = ["--add", "-a", "--del", "--rm", "-r", "--mod", "--edit", "-e", "--show", "-s"] if len(argv) >= 2 and argv[1] == "profile" and argv[0] in profilecmds: argv[1] = argv[0] @@ -291,27 +302,55 @@ class connapp: if args.data == None: print("Missing argument node") exit(3) - matches = list(filter(lambda k: k == args.data, self.nodes)) + # matches = list(filter(lambda k: k == args.data, self.nodes)) + matches = self.config._getallnodes(args.data) if len(matches) == 0: - print("{} not found".format(args.data)) + print("No connection found with filter: {}".format(args.data)) exit(2) - node = self.config.getitem(matches[0]) + elif len(matches) == 1: + uniques = self.config._explode_unique(args.data) + unique = matches[0] + else: + uniques = {"id": None, "folder": None} + unique = None + print("Editing: {}".format(matches)) + node = {} + for i in matches: + node[i] = self.config.getitem(i) edits = self._questions_edit() if edits == None: exit(7) - uniques = self.config._explode_unique(args.data) - updatenode = self._questions_nodes(args.data, uniques, edit=edits) + updatenode = self._questions_nodes(unique, uniques, edit=edits) if not updatenode: exit(7) - uniques.update(node) - uniques["type"] = "connection" - if sorted(updatenode.items()) == sorted(uniques.items()): - print("Nothing to do here") - return + if len(matches) == 1: + uniques.update(node[matches[0]]) + uniques["type"] = "connection" + if sorted(updatenode.items()) == sorted(uniques.items()): + print("Nothing to do here") + return + else: + self.config._connections_add(**updatenode) + self.config._saveconfig(self.config.file) + print("{} edited succesfully".format(args.data)) else: - self.config._connections_add(**updatenode) + for k in node: + updatednode = self.config._explode_unique(k) + updatednode["type"] = "connection" + updatednode.update(node[k]) + editcount = 0 + for key, should_edit in edits.items(): + if should_edit: + editcount += 1 + updatednode[key] = updatenode[key] + if not editcount: + print("Nothing to do here") + return + else: + self.config._connections_add(**updatednode) self.config._saveconfig(self.config.file) - print("{} edited succesfully".format(args.data)) + print("{} edited succesfully".format(matches)) + return def _func_profile(self, args): @@ -526,6 +565,57 @@ class connapp: self.config._saveconfig(self.config.file) print("Config saved") + def _func_import(self, args): + if not os.path.exists(args.data[0]): + print("File {} dosn't exists".format(args.data[0])) + exit(14) + question = [inquirer.Confirm("import", message="Are you sure you want to import {} file? This could overwrite your current configuration".format(args.data[0]))] + confirm = inquirer.prompt(question) + if confirm == None: + exit(7) + if confirm["import"]: + try: + with open(args.data[0]) as file: + imported = yaml.load(file, Loader=yaml.FullLoader) + except: + print("failed reading file {}".format(args.data[0])) + exit(10) + for k,v in imported.items(): + uniques = self.config._explode_unique(k) + folder = f"@{uniques['folder']}" + matches = list(filter(lambda k: k == folder, self.folders)) + if len(matches) == 0: + uniquefolder = self.config._explode_unique(folder) + self.config._folder_add(**uniquefolder) + if "subfolder" in uniques: + subfolder = f"@{uniques['subfolder']}@{uniques['folder']}" + matches = list(filter(lambda k: k == subfolder, self.folders)) + if len(matches) == 0: + uniquesubfolder = self.config._explode_unique(subfolder) + self.config._folder_add(**uniquesubfolder) + uniques.update(v) + self.config._connections_add(**uniques) + self.config._saveconfig(self.config.file) + print("File {} imported succesfully".format(args.data[0])) + return + + def _func_export(self, args): + matches = list(filter(lambda k: k == args.data[0], self.folders)) + if len(matches) == 0: + print("{} folder not found".format(args.data[0])) + exit(2) + if os.path.exists(args.data[1]): + print("File {} already exists".format(args.data[1])) + exit(14) + else: + foldercons = self.config._getallnodesfull(args.data[0], extract = False) + with open(args.data[1], "w") as file: + yaml.dump(foldercons, file, Dumper=NoAliasDumper, default_flow_style=False) + file.close() + print("File {} generated succesfully".format(args.data[1])) + exit() + return + def _func_run(self, args): if len(args.data) > 1: args.action = "noderun" @@ -916,7 +1006,7 @@ class connapp: if "tags" not in defaults: defaults["tags"] = "" except: - defaults = { "host":"", "protocol":"", "port":"", "user":"", "options":"", "logs":"" , "tags":""} + defaults = { "host":"", "protocol":"", "port":"", "user":"", "options":"", "logs":"" , "tags":"", "password":""} node = {} if edit == None: edit = { "host":True, "protocol":True, "port":True, "user":True, "password": True,"options":True, "logs":True, "tags":True } @@ -1082,7 +1172,7 @@ class connapp: if type == "usage": return "conn [-h] [--add | --del | --mod | --show | --debug] [node|folder]\n conn {profile,move,mv,copy,cp,list,ls,bulk,config} ..." if type == "end": - return "Commands:\n profile Manage profiles\n move (mv) Move node\n copy (cp) Copy node\n list (ls) List profiles, nodes or folders\n bulk Add nodes in bulk\n run Run scripts or commands on nodes\n config Manage app config\n api Start and stop connpy api\n ai Make request to an AI" + return "Commands:\n profile Manage profiles\n move (mv) Move node\n copy (cp) Copy node\n list (ls) List profiles, nodes or folders\n bulk Add nodes in bulk\n export Export connection folder to Yaml file\n import Import connection folder to config from Yaml file\n run Run scripts or commands on nodes\n config Manage app config\n api Start and stop connpy api\n ai Make request to an AI" if type == "bashcompletion": return ''' #Here starts bash completion for conn diff --git a/docs/connpy/index.html b/docs/connpy/index.html index 5a3b2bd..7ce2e01 100644 --- a/docs/connpy/index.html +++ b/docs/connpy/index.html @@ -58,6 +58,8 @@ Commands: copy (cp) Copy node list (ls) List profiles, nodes or folders bulk Add nodes in bulk + export Export connection folder to Yaml file + import Import connection folder to config from Yaml file run Run scripts or commands on nodes config Manage app config api Start and stop connpy api @@ -290,6 +292,8 @@ Commands: copy (cp) Copy node list (ls) List profiles, nodes or folders bulk Add nodes in bulk + export Export connection folder to Yaml file + import Import connection folder to config from Yaml file run Run scripts or commands on nodes config Manage app config api Start and stop connpy api @@ -1493,7 +1497,7 @@ Categorize the user's request based on the operation they want to perform on ### Parameters: - - uniques (str/list): Regex string name that will match hostnames + - uniques (str/list): String name that will match hostnames from the connection manager. It can be a list of strings. @@ -1504,6 +1508,8 @@ Categorize the user's request based on the operation they want to perform on ''' nodes = {} + if isinstance(uniques, str): + uniques = [uniques] for i in uniques: if isinstance(i, dict): name = list(i.keys())[0] @@ -1593,7 +1599,7 @@ Categorize the user's request based on the operation they want to perform on raise ValueError("filter must be a string or a list of strings") return nodes - def _getallnodesfull(self, filter = None): + def _getallnodesfull(self, filter = None, extract = True): #get all nodes on configfile with all their attributes. nodes = {} layer1 = {k:v for k,v in self.connections.items() if isinstance(v, dict) and v["type"] == "connection"} @@ -1613,19 +1619,20 @@ Categorize the user's request based on the operation they want to perform on nodes = {k: v for k, v in nodes.items() if any(re.search(pattern, k) for pattern in filter)} else: raise ValueError("filter must be a string or a list of strings") - for node, keys in nodes.items(): - for key, value in keys.items(): - profile = re.search("^@(.*)", str(value)) - if profile: - try: - nodes[node][key] = self.profiles[profile.group(1)][key] - except: - nodes[node][key] = "" - elif value == '' and key == "protocol": - try: - nodes[node][key] = config.profiles["default"][key] - except: - nodes[node][key] = "ssh" + if extract: + for node, keys in nodes.items(): + for key, value in keys.items(): + profile = re.search("^@(.*)", str(value)) + if profile: + try: + nodes[node][key] = self.profiles[profile.group(1)][key] + except: + nodes[node][key] = "" + elif value == '' and key == "protocol": + try: + nodes[node][key] = config.profiles["default"][key] + except: + nodes[node][key] = "ssh" return nodes @@ -1740,7 +1747,7 @@ Categorize the user's request based on the operation they want to perform on

Get a group of nodes from configfile which can be passed to node/nodes class

Parameters:

-
- uniques (str/list): Regex string name that will match hostnames 
+
- uniques (str/list): String name that will match hostnames 
                       from the connection manager. It can be a 
                       list of strings.
 
@@ -1758,7 +1765,7 @@ Categorize the user's request based on the operation they want to perform on ### Parameters: - - uniques (str/list): Regex string name that will match hostnames + - uniques (str/list): String name that will match hostnames from the connection manager. It can be a list of strings. @@ -1769,6 +1776,8 @@ Categorize the user's request based on the operation they want to perform on ''' nodes = {} + if isinstance(uniques, str): + uniques = [uniques] for i in uniques: if isinstance(i, dict): name = list(i.keys())[0] @@ -1883,6 +1892,14 @@ Categorize the user's request based on the operation they want to perform on bulkparser = subparsers.add_parser("bulk", help="Add nodes in bulk") bulkparser.add_argument("bulk", const="bulk", nargs=0, action=self._store_type, help="Add nodes in bulk") bulkparser.set_defaults(func=self._func_others) + # EXPORTPARSER + exportparser = subparsers.add_parser("export", help="Export connection folder to Yaml file") + exportparser.add_argument("export", nargs=2, action=self._store_type, help="export [@subfolder]@folder file.yaml", type=self._type_node) + exportparser.set_defaults(func=self._func_export) + # IMPORTPARSER + importparser = subparsers.add_parser("import", help="Import connection folder to config from Yaml file") + importparser.add_argument("import", nargs=1, action=self._store_type, help="import file.yaml", type=self._type_node) + importparser.set_defaults(func=self._func_import) # AIPARSER aiparser = subparsers.add_parser("ai", help="Make request to an AI") aiparser.add_argument("ask", nargs='*', help="Ask connpy AI something") @@ -1916,7 +1933,7 @@ Categorize the user's request based on the operation they want to perform on configcrud.add_argument("--openai-model", dest="model", nargs=1, action=self._store_type, help="Set openai model", metavar="MODEL") configparser.set_defaults(func=self._func_others) #Manage sys arguments - commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai"] + commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai", "export", "import"] profilecmds = ["--add", "-a", "--del", "--rm", "-r", "--mod", "--edit", "-e", "--show", "-s"] if len(argv) >= 2 and argv[1] == "profile" and argv[0] in profilecmds: argv[1] = argv[0] @@ -2072,27 +2089,55 @@ Categorize the user's request based on the operation they want to perform on if args.data == None: print("Missing argument node") exit(3) - matches = list(filter(lambda k: k == args.data, self.nodes)) + # matches = list(filter(lambda k: k == args.data, self.nodes)) + matches = self.config._getallnodes(args.data) if len(matches) == 0: - print("{} not found".format(args.data)) + print("No connection found with filter: {}".format(args.data)) exit(2) - node = self.config.getitem(matches[0]) + elif len(matches) == 1: + uniques = self.config._explode_unique(args.data) + unique = matches[0] + else: + uniques = {"id": None, "folder": None} + unique = None + print("Editing: {}".format(matches)) + node = {} + for i in matches: + node[i] = self.config.getitem(i) edits = self._questions_edit() if edits == None: exit(7) - uniques = self.config._explode_unique(args.data) - updatenode = self._questions_nodes(args.data, uniques, edit=edits) + updatenode = self._questions_nodes(unique, uniques, edit=edits) if not updatenode: exit(7) - uniques.update(node) - uniques["type"] = "connection" - if sorted(updatenode.items()) == sorted(uniques.items()): - print("Nothing to do here") - return + if len(matches) == 1: + uniques.update(node[matches[0]]) + uniques["type"] = "connection" + if sorted(updatenode.items()) == sorted(uniques.items()): + print("Nothing to do here") + return + else: + self.config._connections_add(**updatenode) + self.config._saveconfig(self.config.file) + print("{} edited succesfully".format(args.data)) else: - self.config._connections_add(**updatenode) + for k in node: + updatednode = self.config._explode_unique(k) + updatednode["type"] = "connection" + updatednode.update(node[k]) + editcount = 0 + for key, should_edit in edits.items(): + if should_edit: + editcount += 1 + updatednode[key] = updatenode[key] + if not editcount: + print("Nothing to do here") + return + else: + self.config._connections_add(**updatednode) self.config._saveconfig(self.config.file) - print("{} edited succesfully".format(args.data)) + print("{} edited succesfully".format(matches)) + return def _func_profile(self, args): @@ -2307,6 +2352,57 @@ Categorize the user's request based on the operation they want to perform on self.config._saveconfig(self.config.file) print("Config saved") + def _func_import(self, args): + if not os.path.exists(args.data[0]): + print("File {} dosn't exists".format(args.data[0])) + exit(14) + question = [inquirer.Confirm("import", message="Are you sure you want to import {} file? This could overwrite your current configuration".format(args.data[0]))] + confirm = inquirer.prompt(question) + if confirm == None: + exit(7) + if confirm["import"]: + try: + with open(args.data[0]) as file: + imported = yaml.load(file, Loader=yaml.FullLoader) + except: + print("failed reading file {}".format(args.data[0])) + exit(10) + for k,v in imported.items(): + uniques = self.config._explode_unique(k) + folder = f"@{uniques['folder']}" + matches = list(filter(lambda k: k == folder, self.folders)) + if len(matches) == 0: + uniquefolder = self.config._explode_unique(folder) + self.config._folder_add(**uniquefolder) + if "subfolder" in uniques: + subfolder = f"@{uniques['subfolder']}@{uniques['folder']}" + matches = list(filter(lambda k: k == subfolder, self.folders)) + if len(matches) == 0: + uniquesubfolder = self.config._explode_unique(subfolder) + self.config._folder_add(**uniquesubfolder) + uniques.update(v) + self.config._connections_add(**uniques) + self.config._saveconfig(self.config.file) + print("File {} imported succesfully".format(args.data[0])) + return + + def _func_export(self, args): + matches = list(filter(lambda k: k == args.data[0], self.folders)) + if len(matches) == 0: + print("{} folder not found".format(args.data[0])) + exit(2) + if os.path.exists(args.data[1]): + print("File {} already exists".format(args.data[1])) + exit(14) + else: + foldercons = self.config._getallnodesfull(args.data[0], extract = False) + with open(args.data[1], "w") as file: + yaml.dump(foldercons, file, Dumper=NoAliasDumper, default_flow_style=False) + file.close() + print("File {} generated succesfully".format(args.data[1])) + exit() + return + def _func_run(self, args): if len(args.data) > 1: args.action = "noderun" @@ -2697,7 +2793,7 @@ Categorize the user's request based on the operation they want to perform on if "tags" not in defaults: defaults["tags"] = "" except: - defaults = { "host":"", "protocol":"", "port":"", "user":"", "options":"", "logs":"" , "tags":""} + defaults = { "host":"", "protocol":"", "port":"", "user":"", "options":"", "logs":"" , "tags":"", "password":""} node = {} if edit == None: edit = { "host":True, "protocol":True, "port":True, "user":True, "password": True,"options":True, "logs":True, "tags":True } @@ -2863,7 +2959,7 @@ Categorize the user's request based on the operation they want to perform on if type == "usage": return "conn [-h] [--add | --del | --mod | --show | --debug] [node|folder]\n conn {profile,move,mv,copy,cp,list,ls,bulk,config} ..." if type == "end": - return "Commands:\n profile Manage profiles\n move (mv) Move node\n copy (cp) Copy node\n list (ls) List profiles, nodes or folders\n bulk Add nodes in bulk\n run Run scripts or commands on nodes\n config Manage app config\n api Start and stop connpy api\n ai Make request to an AI" + return "Commands:\n profile Manage profiles\n move (mv) Move node\n copy (cp) Copy node\n list (ls) List profiles, nodes or folders\n bulk Add nodes in bulk\n export Export connection folder to Yaml file\n import Import connection folder to config from Yaml file\n run Run scripts or commands on nodes\n config Manage app config\n api Start and stop connpy api\n ai Make request to an AI" if type == "bashcompletion": return ''' #Here starts bash completion for conn @@ -3050,7 +3146,7 @@ tasks:
-def start(self, argv=['connpy', '-o', 'docs/', '--html', '--force']) +def start(self, argv=['connpy', '-o', 'docs/', '--force', '--html'])

Parameters:

@@ -3109,6 +3205,14 @@ tasks: bulkparser = subparsers.add_parser("bulk", help="Add nodes in bulk") bulkparser.add_argument("bulk", const="bulk", nargs=0, action=self._store_type, help="Add nodes in bulk") bulkparser.set_defaults(func=self._func_others) + # EXPORTPARSER + exportparser = subparsers.add_parser("export", help="Export connection folder to Yaml file") + exportparser.add_argument("export", nargs=2, action=self._store_type, help="export [@subfolder]@folder file.yaml", type=self._type_node) + exportparser.set_defaults(func=self._func_export) + # IMPORTPARSER + importparser = subparsers.add_parser("import", help="Import connection folder to config from Yaml file") + importparser.add_argument("import", nargs=1, action=self._store_type, help="import file.yaml", type=self._type_node) + importparser.set_defaults(func=self._func_import) # AIPARSER aiparser = subparsers.add_parser("ai", help="Make request to an AI") aiparser.add_argument("ask", nargs='*', help="Ask connpy AI something") @@ -3142,7 +3246,7 @@ tasks: configcrud.add_argument("--openai-model", dest="model", nargs=1, action=self._store_type, help="Set openai model", metavar="MODEL") configparser.set_defaults(func=self._func_others) #Manage sys arguments - commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai"] + commands = ["node", "profile", "mv", "move","copy", "cp", "bulk", "ls", "list", "run", "config", "api", "ai", "export", "import"] profilecmds = ["--add", "-a", "--del", "--rm", "-r", "--mod", "--edit", "-e", "--show", "-s"] if len(argv) >= 2 and argv[1] == "profile" and argv[0] in profilecmds: argv[1] = argv[0]