#!/usr/bin/env python import os import pwd import sys import subprocess from P4 import P4, P4Exception sys.path.append("/usr/local/lib/netapp-manageability-sdk-5.3.1/lib/python/NetApp") from NaServer import * if sys.version_info[0] >= 3: from configparser import ConfigParser else: from ConfigParser import ConfigParser # --------------------------------------------------------------------------- # Singleton class (of sorts) for loading configuration # --------------------------------------------------------------------------- class Config: def __init__(self, file): self.config = ConfigParser() try: self.config.read(file) except: print("Could not read config file: %s" % file) sys.exit(2) def get(self, key): parts = key.split(".") try: value = self.config.get(parts[0], parts[1]) except: print("Could not access config: %s" % key) sys.exit(2) return value dir = os.path.dirname(os.path.realpath(__file__)) config = Config(dir + "/flex.cfg") FLEX_SNAP = config.get("p4.snap") FLEX_CLONE = config.get("p4.clone") # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # NetApp Server connection as 'admin' user # --------------------------------------------------------------------------- class NaFlex: def __init__(self): self.server = config.get("NaServer.server") self.port = config.get("NaServer.port") self.user = config.get("NaServer.admin_user") self.passwd = config.get("NaServer.admin_passwd") self.vserver = config.get("NaServer.vserver") self.transport = config.get("NaServer.transport") self.style = config.get("NaServer.style") self.server_type = config.get("NaServer.server_type") self.aggr = config.get("NaServer.aggr") self.mount = config.get("NaServer.mount_base") # Sets the filer properties for connection # @returns the NaServer connection def get(self): s = NaServer(self.server, 1 , 15) s.set_server_type(self.server_type) resp = s.set_style(self.style) if (resp and resp.results_errno() != 0) : r = resp.results_reason() print ("Failed to set authentication style " + r + "\n") sys.exit (2) s.set_admin_user(self.user, self.passwd) resp = s.set_transport_type(self.transport) if (resp and resp.results_errno() != 0) : r = resp.results_reason() print ("Unable to set HTTP transport " + r + "\n") sys.exit (2) s.set_vserver(self.vserver) s.set_port(self.port) return s # Creates the volume based on name and size def create(self, name, size): api = NaElement("volume-create") api.child_add_string("containing-aggr-name", self.aggr) api.child_add_string("size", size) api.child_add_string("volume", name) # api.child_add_string("junction-active", "true") jpath = "/" + name api.child_add_string("junction-path", jpath) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Takes a snapshot of specified volume def snapshot(self, volname, snapname): api = NaElement("snapshot-create") api.child_add_string("volume", volname) api.child_add_string("snapshot", snapname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Creates a volume clone based on the snapshot of parent volume def clone(self, cvolname, psnapname, pvolname): api = NaElement("volume-clone-create") api.child_add_string("junction-active", "true") jpath = "/" + cvolname api.child_add_string("junction-path", jpath) api.child_add_string("parent-snapshot", psnapname) api.child_add_string("parent-volume", pvolname) api.child_add_string("volume", cvolname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Deletes the volume by unmounting, offlining and then deleting volume from vserver def delete(self, volname): # Check to make sure we own the volume to delete # st = os.stat(self.mount + "/" + volname) # user = self.call.getUser() # uid = pwd.getpwname(user).pw_uid # gid = pwd.getpwname(user).pw_gid # if (uid != st.st_uid): # raise NAException("You do not have permission to delete volume. Owner of volume is %s." % (pwd.getpwuid(st.st_uid).pw_name)) # Check to make sure volume passed in is not vserver root or node volume # Root and node volumes are required to keep info on state of vserver and node api = NaElement("volume-get-iter") # Build the input values for the api query call xi1 = NaElement("query") api.child_add(xi1) xi2 = NaElement("volume-attributes") xi1.child_add(xi2) xi3 = NaElement("volume-id-attributes") xi2.child_add(xi3) xi3.child_add_string("name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) vlist = xo.child_get("attributes-list") vsattrs = None if (vlist != None): vol = vlist.child_get("volume-attributes") vsattrs = vol.child_get("volume-state-attributes") if (vsattrs != None): isvroot = vsattrs.child_get_string("is-vserver-root") isnroot = vsattrs.child_get_string("is-node-root") if ((isvroot == "true") or (isnroot == "true")): raise NAException("Not authorized to delete vserver root-volume %s. Go directly to the NetApp Filer to conduct this operation" % (volname)) # Unmount api = NaElement("volume-unmount") api.child_add_string("force", "false") api.child_add_string("volume-name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Offline api = NaElement("volume-offline") api.child_add_string("name", volname) xo = self.get().invoke_elem(api) if ((xo.results_status() == "failed") and (xo.results_errno() != "13042")): raise NAException(xo.sprintf()) # Delete api = NaElement("volume-destroy") api.child_add_string("name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Prints out the list of volumes def vlist (self): list = [] # Get First Set of mrecs (100) records mrecs = "100" api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up to get next set of records tag = xo.child_get_string("next-tag") # Add vollist results to list and then Get Following set of records # Break from loop when the vollist is None or next-tag is None while True: vollist = xo.child_get("attributes-list") if (vollist == None): break for vol in vollist.children_get(): volattrs = vol.child_get("volume-id-attributes") volpname = "" volstateattrs = vol.child_get("volume-state-attributes") if (volstateattrs != None): state = volstateattrs.child_get_string("state") isroot = volstateattrs.child_get_string("is-vserver-root") volcattrs = vol.child_get("volume-clone-attributes") if (volcattrs != None): volpattrs = volcattrs.child_get("volume-clone-parent-attributes") if (volpattrs != None): volpname = volpattrs.child_get_string("name") # Only print out volume if volume is online, not a clone, and not vserver root volume if (state == "online") and (volpname == "") and (isroot != "true"): list.append(volattrs); api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Get next tag which indicates if there are more records tag = xo.child_get_string("next-tag") if (tag == None): break return list # Print out list of snapshots based on volume name def slist(self, vname): # Get First Set of records api = NaElement("snapshot-list-info") api.child_add_string("volume", vname) xo = self.get().invoke_elem(api) if (xo.results_errno() != 0) : raise NAException(xo.sprintf()) # # get snapshot list snapshotlist = xo.child_get("snapshots") print("\n") if ((snapshotlist != None) and (snapshotlist != "")) : # iterate through snapshot list snapshots = snapshotlist.children_get() for ss in snapshots: sname = ss.child_get_string("name") print("snapshot:%s" % (sname)) else: print("No snapshots for volume " + vname) # Print out volume name and their snapshots def slistall(self): # Get first set of records defined by mrecs mrecs = "100" api = NaElement("snapshot-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up tag to get next set of records tag = xo.child_get_string("next-tag") # Need to go thru this loop at least once before checking for next tag # Break out of loop if snaplist is None or next-tag is None while True: # Get list of snapshot-info attributes snaplist = dict() snaplist = xo.child_get("attributes-list") if (snaplist == None): break # Go thru list and print out volume and snapshot name for snap in snaplist.children_get(): print("snapshot: %s:%s" % (snap.child_get_string("volume"), snap.child_get_string("name"))) # Get next set of records api = NaElement("snapshot-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Tag indicates if there are more records tag = xo.child_get_string("next-tag") # Break out of loop. Tag of None indicates no more records if (tag == None): break # # Prints out list of clones with their corresponding parent volume and parent snapshot def clist(self): # Get first set number of records defined by mrecs mrecs = "100" api = NaElement("volume-clone-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up to get next set of records tag = xo.child_get_string("next-tag") print("\n") # Need to go thru this loop at least once before checking for next tag while True: # Get list of snapshot-info attributes clonelist = dict() clonelist = xo.child_get("attributes-list") if (clonelist == None): break for clone in clonelist.children_get(): print("clone: %s:%s:%s" % (clone.child_get_string("parent-volume"), clone.child_get_string("parent-snapshot"), clone.child_get_string("volume"))) # Get next set of records api = NaElement("volume-clone-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Tag indicates if there are more records tag = xo.child_get_string("next-tag") # Break if no more records if (tag == None): break class NAException(Exception): def __init__(self, e): self.error = e # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Perforce connection as 'flex' user # --------------------------------------------------------------------------- class P4Flex: def __init__(self): self.port = config.get("p4.port") self.user = config.get("p4.user") self.passwd = config.get("p4.passwd") def getP4(self): p4 = P4() p4.user = self.user p4.port = self.port p4.connect() p4.password = self.passwd p4.run_login() return p4 # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Parse the Broker's request and provide access to the user's environment # --------------------------------------------------------------------------- class Broker: def __init__(self): self.args = {} # Comment out stdin and uncomment block below for debug. lines = sys.stdin.readlines() # lines = [] # with open("in.txt") as f: # lines = f.readlines() for line in lines: parts = line.split(": ") self.args[parts[0]] = parts[1].rstrip() def getP4(self): p4 = P4() p4.user = self.args['user'] p4.port = self.args['clientPort'] p4.client = self.args['workspace'] p4.connect() # Use flex account to login user s4 = P4Flex().getP4() s4.run_login(self.args['user']) return p4 def getPort(self): return self.args['clientPort'] def getUser(self): return self.args['user'] def getClient(self): return self.args['workspace'] def getCommand(self): if 'Arg0' in self.args: return self.args['Arg0'] else: return None def getOptions(self): c = 1 opts = [] while True: key = 'Arg' + str(c) c += 1 if key in self.args: opts.append(self.args[key]) else: break join = "" list = [] for opt in opts: if join: opt = join + opt join = "" if opt.startswith('-') and len(opt) == 2: join = opt else: list.append(opt) if join: list.append(opt) return list # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Handle Broker request and invoke the registered 'flex' command # --------------------------------------------------------------------------- class Flex: def __init__(self): # Process Broker arguments self.call = Broker() self.command = self.call.getCommand() self.args = self.call.args self.opts = self.call.getOptions() # Build table for function pointers self.cmdFn = { 'volume': self.volume, 'volumes': self.volumes, 'snapshot': self.snapshot, 'snapshots': self.snapshots, 'clone': self.clone, 'clones': self.clones, 'help': self.help }; if not self.isnfs(NaFlex().mount): print("action: REJECT") print("message: \"%s\"" % "NetApp nfs volume is not mounted.") return # Call command if self.command in self.cmdFn: self.cmdFn[self.command]() else: self.usage() def volume(self): # Check for 'admin' permission if not self.permission(): print("action: REJECT") print("message: \"%s\"" % "You don't have permission for this operation.") return # Get options (set defaults) vname = "" size = "1G" for o in self.opts: if o.startswith('-d'): del_name = o[2:] self.vol_del(del_name) return if o.startswith('-s'): size = o[2:] else: vname = o if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return # NetApp connection netapp = NaFlex() # Create NetApp volume try: netapp.create(vname, size) # Set file ownership of newly created volume. user = self.call.getUser() path = netapp.mount + "/" + vname self.chown(user, path) msg = vname + ". Mounted on " + path + "." print("action: RESPOND") print("message: \"Created flex volume %s\"" % msg) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) except Exception as e: # If got unexpected error, delete the volume created error = "Unexpected Error: " + str(e) try: netapp.delete(vname) except NAException as e: error += '\nNetApp Error: ' + e.error print("action: RESPOND") print("message: \"%s\"" % error) def volumes(self): # NetApp connection netapp = NaFlex() # List the volumes try: list = "" vols = netapp.vlist() for v in vols: list += "volume '" list += v.child_get_string("name") + "' " #list += v.child_get_string("name-ordinal") + " " #list += v.child_get_string("owning-vserver-name") + " " #list += "'" + v.child_get_string("comment")[:50] + "'" list += "\n" print("action: RESPOND") print("message: \"%s\"" % list) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) # Builds a new FLEX_SNAP workspace to create the snapshot def snapshot(self): # Get options (set defaults) vname = "" sname = "" from_client_name = self.call.getClient() for o in self.opts: if o.startswith('-V'): vname = o[2:] if o.startswith('-c'): from_client_name = o[2:] else: sname = o if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return if not sname: print("action: REJECT") print("message: \"%s\"" % "No flex clone name provided") return if ':' in sname: print("action: REJECT") print("message: \"%s\"" % "Flex clone name must not use ':'") return if not self.permission(): print("action: REJECT") print("message: \"%s\"" % "You don't have permission for this operation.") return # NetApp/Perforce connection as Flex netapp = NaFlex() p4 = P4Flex().getP4() # Create NetApp snapshot try: netapp.snapshot(vname, sname) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) # Create Perforce workspace for snapshot try: from_client = p4.fetch_client(from_client_name) root = from_client['Root'] # Clone client workspace flex_client_name = FLEX_SNAP + vname + ':' + sname p4.client = flex_client_name flex_client = p4.fetch_client("-t", from_client_name, flex_client_name) # Set workspace options: root to mounted volume path = netapp.mount + "/" + vname flex_client['Root'] = path flex_client['Host'] = "" #flex_client['Options'] = flex_client['Options'].replace(" unlocked", " locked") p4.save_client(flex_client) # Populate have list p4.run_sync("-k", "//" + flex_client_name + "/...@" + from_client_name) print("action: RESPOND") print("message: \"Created flex snapshot %s\"" % sname) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def permission(self): try: # Perforce connection as Caller p4 = self.call.getP4() for p in p4.run_protects(): if (p['perm'] == 'admin') or (p['perm'] == 'super'): return True return False except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def snapshots(self): try: # Perforce connection as Caller p4 = self.call.getP4() snapshots = p4.run_clients("-e", FLEX_SNAP + "*") list = "" for v in snapshots: name = v['client'][len(FLEX_SNAP):] part = name.split(':', 1) list += "snapshot '" + part[1] + "' volume '" + part[0] + "' root '" + v['Root'] + "'\n" print("action: RESPOND") print("message: \"%s\"" % list) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() # Clones a new FLEX_CLONE workspace from the parent flexSnap_ def clone(self): # Get options vname = "" sname = "" cname = "" for o in self.opts: if o.startswith('-d'): del_name = o[2:] self.clone_del(del_name) return if o.startswith('-V'): vname = o[2:] if o.startswith('-S'): sname = o[2:] else: cname = o if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return if not sname: print("action: REJECT") print("message: \"%s\"" % "No flex snapshot name provided") return if not cname: print("action: REJECT") print("message: \"%s\"" % "No flex clone name provided") return if ':' in cname: print("action: REJECT") print("message: \"%s\"" % "Flex clone name must not use ':'") return # NetApp/Perforce connection as Caller netapp = NaFlex() p4 = self.call.getP4() path = netapp.mount + "/" + cname # Create NetApp clone from snapshot try: netapp.clone(cname, sname, vname) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) try: # Verify parent client exists parent_client_name = FLEX_SNAP + vname + ":" + sname list = p4.run_clients("-e", parent_client_name) if len(list) != 1: print("action: REJECT") print("message: \"Flex parent %s does not exist.\"" % parent_client_name) return # Clone client workspace clone_client_name = FLEX_CLONE + cname p4.client = clone_client_name clone_client = p4.fetch_client("-t", parent_client_name, clone_client_name) clone_client['Root'] = path #clone_client['Options'] = clone_client['Options'].replace(" unlocked", " locked") p4.save_client(clone_client) # Generate P4CONFIG file self.p4config(path, clone_client_name) # Populate have list p4.run_sync("-k", "//" + clone_client_name + "/...@" + parent_client_name) # Set file ownership user = self.call.getUser() self.chown(user, path) msg = clone_client_name + ". Mounted on " + path + "." print("action: RESPOND") print("message: \"Created flex clone client %s\"" % msg) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def chown(self, user, path): # Checks need to be done in case of failure uid = pwd.getpwnam(user).pw_uid gid = pwd.getpwnam(user).pw_gid os.system('sudo chown -R ' + str(uid) + ':' + str(gid) + ' ' + path) def isnfs(self, path): # Check if path is nfs mounted. fstype = subprocess.check_output('stat -f -L -c %T ' + path, shell=True) if (fstype == 'nfs\n'): return True return False def clones(self): # Get options all = False for o in self.opts: if o == "-a": all = True try: # Perforce connection as Caller p4 = self.call.getP4() clones = [] if all: clones = p4.run_clients("-e", FLEX_CLONE + "*") else: user = self.call.getUser() clones = p4.run_clients("-e", FLEX_CLONE + "*", "-u", user) list = "" for c in clones: name = c['client'][len(FLEX_CLONE):] list += "clone '" + name + "' owner '" + c['Owner'] + "' root '" + c['Root'] + "'\n" print("action: RESPOND") print("message: \"%s\"" % list) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def clone_del(self, cname): if not cname: print("action: REJECT") print("message: \"%s\"" % "No Flex clone name provided") return # NetApp/Perforce connection as Caller netapp = NaFlex() p4 = self.call.getP4() # Delete Perforce workspace try: client_name = FLEX_CLONE + cname p4.run_client("-d", client_name) print("action: REJECT") print("message: \"Deleted Flex clone %s\"" % cname) except P4Exception: print("action: REJECT") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) return finally: p4.disconnect() # If client delete succeed, Delete NetApp clone try: netapp.delete(cname) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message: \"%s\"" % error) return def vol_del(self, vname): if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return # NetApp/Perforce connection as Flex netapp = NaFlex() p4 = P4Flex().getP4() # Delete Perforce workspaces try: clones = p4.run_clients("-e", FLEX_SNAP + vname + "*") list = "" for c in clones: client_name = c['client'] p4.run_client("-f", "-d", client_name) list += " deleted client: %s\n" % client_name print("action: RESPOND") message = "Deleted Flex volume %s\n" % vname message += list print("message: \"%s\"" % message) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() # If client delete succeed, Delete NetApp volume try: netapp.delete(vname) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message: \"%s\"" % error) return def p4config(self, path, client): p4config = os.getenv('P4CONFIG', '.p4config') p4port = self.call.getPort() p4user = self.call.getUser() fh = open(path + "/" + p4config, "w") fh.write("# Generated by p4 flex.\n"); fh.write("P4PORT=" + p4port + "\n"); fh.write("P4USER=" + p4user + "\n"); fh.write("P4CLIENT=" + client + "\n"); fh.close() def help(self): help = ( "(FlexClone)\n\n" " flex -- Perforce FlexClone operations\n\n" " p4 flex volume -s size[M, G] name\n" " p4 flex volume -d name\n" " p4 flex volumes \n\n" " p4 flex snapshot -V volume [-c client] name\n" " p4 flex snapshots \n\n" " p4 flex clone -V volume -S parent name\n" " p4 flex clone -d name\n" " p4 flex clones [-a]\n" " \n" " 'p4 flex volume' will create a new volume.The '-d' flag will\n" " delete the volume and associated snapshots.\n" " \n" " 'p4 flex volumes' will display a list of all volumes.\n" " \n" " 'p4 flex snapshot' will create a new snapshot.\n" " \n" " 'p4 flex snapshots' will display a list of all flex snapshots.\n" " \n" " 'p4 flex clone' will create a new flex clone. The '-d' flag will\n" " delete the flex clone and associated client.\n" " \n" " 'p4 flex clones' will display a list of all flex clones owned by that\n" " user. The '-a' flag will list all flex clones globally.\n" ) print("action: RESPOND") print("message: \"%s\"" % help) def usage(self): usage = ( "Usage: flex { volume(s) | snapshot(s) | clone(s) }\n" "Missing/wrong number of arguments." ) print("action: RESPOND") print("message: \"%s\"" % usage) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # MAIN # --------------------------------------------------------------------------- if __name__ == '__main__': flex = Flex()