#!/usr/bin/env python import os import pwd import grp import sys import subprocess import pdb from P4 import P4, P4Exception sys.path.append("/usr/local/lib/netapp-manageability-sdk-9.3/lib/python/NetApp") from NaServer import * import ssl try: _create_unverified_https_context = ssl._create_unverified_context except AttributeError: # Legacy Python that doesn't verify HTTPS certificates by default pass else: # Handle target environment that doesn't support HTTPS verification ssl._create_default_https_context = _create_unverified_https_context if sys.version_info[0] >= 3: from configparser import ConfigParser else: from ConfigParser import ConfigParser # --------------------------------------------------------------------------- # Singleton class (of sorts) for loading configuration # --------------------------------------------------------------------------- class Config: def __init__(self, file): self.config = ConfigParser() try: self.config.read(file) except: print("Could not read config file: %s" % file) sys.exit(2) def get(self, key): parts = key.split(".") try: value = self.config.get(parts[0], parts[1]) except: print("Could not access config: %s" % key) sys.exit(2) return value dir = os.path.dirname(os.path.realpath(__file__)) config = Config(dir + "/flex.cfg") FLEX_SNAP = config.get("p4.snap") FLEX_CLONE = config.get("p4.clone") # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # NetApp Server connection as 'admin' user # --------------------------------------------------------------------------- class NaFlex: def __init__(self): self.server = config.get("NaServer.server") self.port = config.get("NaServer.port") self.user = config.get("NaServer.admin_user") self.passwd = config.get("NaServer.admin_passwd") self.vserver = config.get("NaServer.vserver") self.transport = config.get("NaServer.transport") self.style = config.get("NaServer.style") self.server_type = config.get("NaServer.server_type") self.aggr = config.get("NaServer.aggr") self.mount = config.get("NaServer.mount_base") # Sets the filer properties for connection # @returns the NaServer connection def get(self): s = NaServer(self.server, 1 , 15) s.set_server_type(self.server_type) resp = s.set_style(self.style) if (resp and resp.results_errno() != 0) : r = resp.results_reason() print ("Failed to set authentication style " + r + "\n") sys.exit (2) s.set_admin_user(self.user, self.passwd) resp = s.set_transport_type(self.transport) if (resp and resp.results_errno() != 0) : r = resp.results_reason() print ("Unable to set HTTP transport " + r + "\n") sys.exit (2) s.set_vserver(self.vserver) s.set_port(self.port) return s # Creates the volume based on name and size def create(self, name, size, junction_path, uid, gid): ontapversion = self.getontapversion() api = NaElement("volume-create") api.child_add_string("containing-aggr-name", self.aggr) api.child_add_string("size", size) api.child_add_string("volume", name) # api.child_add_string("junction-active", "true") api.child_add_string("junction-path", junction_path) if "9." in ontapversion: #ru the ontap 9 commands for chowning api.child_add_string("group-id", gid) api.child_add_string("user-id", uid) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) def snapshot_delete(self, volname, snapname): api = NaElement("snapshot-delete") api.child_add_string("volume", volname) api.child_add_string("snapshot", snapname) #api.child_add_string("vserver", "P4FlexServer") # initialize exit message exit_msg = "" # invoke snapshot delete # print ("\nvolume name is %s" %volname) # print ("\nsnapshot name is %s" %snapname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): exit_msg += "ERROR failed to delete snapshot" exit_msg += " Reason:" + xo.results_reason() # return exit message - no message if successful return exit_msg # Takes a snapshot of specified volume def snapshot(self, volname, snapname): api = NaElement("snapshot-create") api.child_add_string("volume", volname) api.child_add_string("snapshot", snapname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Creates a volume clone based on the snapshot of parent volume def clone(self, cvolname, psnapname, pvolname, uid, gid,junction_path): api = NaElement("volume-clone-create") api.child_add_string("junction-active", "true") jpath = junction_path api.child_add_string("junction-path", jpath) api.child_add_string("parent-snapshot", psnapname) api.child_add_string("parent-volume", pvolname) api.child_add_string("volume", cvolname) # force the flexclone to use thin provisioning even if the parent # volume is thick previsioned. without this, the clone will not have # any space savings. it will take the same reserved space as the parent. api.child_add_string("space-reserve", "none") ontapversion = self.getontapversion() # print ("\nhi") # print ("ontap version is %s " %ontapversion) if "9." in ontapversion: #run the ontap 9 commands for chowning api.child_add_string("gid", gid) api.child_add_string("uid", uid) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # def modvolumeown: #A function attempting to modify volume ownership # api = NaElement("volume-clone-create") # api.child_add_string("junction-active", "true") # jpath = "/" + cvolname def ckclone(self, vname): # Build the input values for the api query call #This function checks if a volume has clones - True= has clones api = NaElement("volume-clone-get-iter") # now ck if api has vname as master volume xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) vollist = xo.child_get("attributes-list") if (vollist == None): return "False" for vol_data in vollist.children_get(): clone_name = vol_data.child_get_string( "volume"); parent_volname = vol_data.child_get_string( "parent-volume" ); if parent_volname==vname: return "True" return "False" def getontapversion(self): api = NaElement ("system-get-version") xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) sysversion=xo.child_get_string("version") return sysversion # Deletes the volume by unmounting, offlining and then deleting volume from vserver def delete(self, volname): # Check to make sure we own the volume to delete # st = os.stat(self.mount + "/" + volname) # user = self.call.getUser() # uid = pwd.getpwname(user).pw_uid # gid = pwd.getpwname(user).pw_gid # if (uid != st.st_uid): # raise NAException("You do not have permission to delete volume. Owner of volume is %s." % (pwd.getpwuid(st.st_uid).pw_name)) # Check to make sure volume passed in is not vserver root or node volume # Root and node volumes are required to keep info on state of vserver and node api = NaElement("volume-get-iter") # Build the input values for the api query call xi1 = NaElement("query") api.child_add(xi1) xi2 = NaElement("volume-attributes") xi1.child_add(xi2) xi3 = NaElement("volume-id-attributes") xi2.child_add(xi3) xi3.child_add_string("name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) vlist = xo.child_get("attributes-list") vsattrs = None if (vlist != None): vol = vlist.child_get("volume-attributes") vsattrs = vol.child_get("volume-state-attributes") if (vsattrs != None): isvroot = vsattrs.child_get_string("is-vserver-root") isnroot = vsattrs.child_get_string("is-node-root") if ((isvroot == "true") or (isnroot == "true")): raise NAException("Not authorized to delete vserver root-volume %s. Go directly to the NetApp Filer to conduct this operation" % (volname)) # Unmount api = NaElement("volume-unmount") api.child_add_string("force", "false") api.child_add_string("volume-name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Offline api = NaElement("volume-offline") api.child_add_string("name", volname) xo = self.get().invoke_elem(api) if ((xo.results_status() == "failed") and (xo.results_errno() != "13042")): raise NAException(xo.sprintf()) # Delete api = NaElement("volume-destroy") api.child_add_string("name", volname) xo = self.get().invoke_elem(api) if (xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Prints out the list of volumes def vlist (self): list = [] # Get First Set of mrecs (100) records mrecs = "100" api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up to get next set of records tag = xo.child_get_string("next-tag") # Add vollist results to list and then Get Following set of records # Break from loop when the vollist is None or next-tag is None while True: vollist = xo.child_get("attributes-list") if (vollist == None): break for vol in vollist.children_get(): volattrs = vol.child_get("volume-id-attributes") volpname = "" volstateattrs = vol.child_get("volume-state-attributes") if (volstateattrs != None): state = volstateattrs.child_get_string("state") isroot = volstateattrs.child_get_string("is-vserver-root") volcattrs = vol.child_get("volume-clone-attributes") if (volcattrs != None): volpattrs = volcattrs.child_get("volume-returnclone-parent-attributes") if (volpattrs != None): volpname = volpattrs.child_get_string("name") # Only print out volume if volume is online, not a clone, and not vserver root volume if (state == "online") and (volpname == "") and (isroot != "true"): list.append(volattrs); api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Get next tag which indicates if there are more records tag = xo.child_get_string("next-tag") if (tag == None): break return list # Print out list of snapshots based on volume name def slist(self, vname): # Get First Set of records api = NaElement("snapshot-list-info") api.child_add_string("volume", vname) xo = self.get().invoke_elem(api) if (xo.results_errno() != 0) : raise NAException(xo.sprintf()) # # get snapshot list snapshotlist = xo.child_get("snapshots") if ((snapshotlist != None) and (snapshotlist != "")) : # iterate through snapshot list snapshots = snapshotlist.children_get() for ss in snapshots: sname = ss.child_get_string("name") print("snapshot:%s" % (sname)) else: print("No snapshots for volume " + vname) # Print out volume name and their snapshots def slistall(self): # Get first set of records defined by mrecs mrecs = "100" api = NaElement("snapshot-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up tag to get next set of records tag = xo.child_get_string("next-tag") # Need to go thru this loop at least once before checking for next tag # Break out of loop if snaplist is None or next-tag is None while True: # Get list of snapshot-info attributes snaplist = dict() snaplist = xo.child_get("attributes-list") if (snaplist == None): break # Go thru list and print out volume and snapshot name for snap in snaplist.children_get(): print("snapshot: %s:%s" % (snap.child_get_string("volume"), snap.child_get_string("name"))) # Get next set of records api = NaElement("snapshot-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Tag indicates if there are more records tag = xo.child_get_string("next-tag") # Break out of loop. Tag of None indicates no more records if (tag == None): break def list_flexclones(self, clonelist): footprintdict =dict() mrecs=100 # Get the exact method from volume-footprint-get-iter # get the footprints of all volumes first bpi = NaElement("volume-footprint-get-iter") # bpi.child_add_string("footprint-info", "total-footprint") bpi.child_add_string("max-records", mrecs) mo = self.get().invoke_elem(bpi) # print ("The footprint is %d" %mo.child_get("total-footprint")) if(mo.results_status() == "failed"): raise NAException(xo.sprintf()) tag = mo.child_get_string("next-tag") while True: # # get list of volume-info attributes volattrs = dict() volattrs = mo.child_get("attributes-list") if (volattrs == None): break for part in volattrs.children_get(): vol_name = part.child_get_string("volume") footprint= part.child_get_string("total-footprint") if vol_name : footprintdict[vol_name]=footprint #The footprintlist should be taken to process the output # get next set of records bpi = NaElement("volume-footprint-get-iter") bpi.child_add_string("max-records", mrecs) bpi.child_add_string("tag", tag) mo = self.get().invoke_elem(bpi) if(mo.results_status() == "failed"): raise NAException(xo.sprintf()) # Get next tag which indicates if there are more records tag = mo.child_get_string("next-tag") if (tag == None): break junction_path_map = dict() comment_field_map = dict() vol_usage_map = dict() vol_dedup_saved = dict() vol_dedup_shared = dict() # Get First Set of mrecs (100) records mrecs = "100" api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up to get next set of records tag = xo.child_get_string("next-tag") # Add vollist results to list and then Get Following set of records # Break from loop when the vollist is None or next-tag is None while True: # get list of volume-info attributes vollist = dict() vollist = xo.child_get("attributes-list") if (vollist == None): break # loop thru list of volumes to get specific volume attribute data for tattr in vollist.children_get(): vol_id_attrs = tattr.child_get("volume-id-attributes") if vol_id_attrs: volume_name = vol_id_attrs.child_get_string("name") junct_path = vol_id_attrs.child_get_string("junction-path") # print ("\n junction path is %s " %junct_path) comment_field = vol_id_attrs.child_get_string("comment") # store junction_path for later junction_path_map[volume_name] = junct_path # if the comment field is empty, just put UNKNOWN as the value if comment_field: comment_field_map[volume_name] = comment_field else: comment_field_map[volume_name] = "UNKNOWN" vol_space_attrs = tattr.child_get( "volume-space-attributes" ); if vol_space_attrs: vol_usage = vol_space_attrs.child_get_string( "size-used" ); if vol_usage: vol_usage_map[volume_name] = vol_usage; #print "DEBUG: vol usage: $volume_name $vol_usage_map{$volume_name}\n"; dedup_saved="" vol_sis_attrs = tattr.child_get( "volume-sis-attributes" ) if vol_sis_attrs: dedup_saved = vol_sis_attrs.child_get_string( "percentage-total-space-saved" ) dedup_shared = vol_sis_attrs.child_get_string( "deduplication-space-shared" ) if dedup_saved: vol_dedup_saved[volume_name] = dedup_saved vol_dedup_shared[volume_name] = dedup_shared # get next set of records api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Get next tag which indicates if there are more records tag = xo.child_get_string("next-tag") if (tag == None): break #---------------------------------------- # create report header #---------------------------------------- list = "\nList FlexClones\n"; #123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890 list += "%-25s %-30s %-20s " % ("Parent Volume", "Parent-Snapshot", "FlexClone"); list += "%15s" % "Parent Space Con"; list += "%25s" % "Flexclone Space Con"; # list += "%15s" % "Split Est"; list += "%30s" % "Flexclone % of Parent"; list += " %s \n" % "Linux-Clone-path"; list += "---------------------------------------------------------------------------------------"; list += "---------------------------------------------------------------------------------------------------\n"; #---------------------------------------- # get FlexCone info iteratively - it will return a list #---------------------------------------- # Get First Set of mrecs (100) records mrecs = "100" api = NaElement("volume-clone-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) tag = xo.child_get_string("next-tag") # Add vollist results to list and then Get Following set of records # Break from loop when the vollist is None or next-tag is None while True: # # get list of volume-info attributes #vollist is the orignal volume list - all volumes vollist = dict() vollist = xo.child_get("attributes-list") if (vollist == None): break # for each clone entry # Use the P4 list here to compare against clone_name="somestuffyyyyy" for cpart in clonelist: cname = cpart['client'][len(FLEX_CLONE):] for vol_data in vollist.children_get(): clone_name = vol_data.child_get_string( "volume"); if str(cname)==str(clone_name): volume_name = vol_data.child_get_string( "parent-volume" ); clone_name = vol_data.child_get_string( "volume" ); snapshot = vol_data.child_get_string( "parent-snapshot"); #flexclone_used = vol_data.child_get_string( "used" ); # split_est = vol_data.child_get_string( "split-estimate" ); comment_field = comment_field_map[clone_name]; if comment_field == "": comment_field = "USER_UNKNOWN" # # # parent volume: space used - represent data used in MB vol_usage = footprintdict[volume_name] parent_footprint = float(vol_usage)/1024/1024 # # # FlexClone volume: space used - represent data used in MB clone_usage = footprintdict[clone_name] clone_footprint = float(clone_usage) / 1024 / 1024; # Find how much of the parent volume size is consumed by the clone pctofparent = float(clone_footprint/parent_footprint)*100 # # determine juction-path info jpath = str(self.mount) + vol_data.child_get_string( "junction-path" ); # test if the value returned correctly if jpath: # # perfect the look up worked correctly dummyval = 0; elif junction_path_map[clone_name]: # # ok lookup didn't work, but it was found by method #2 jpath = junction_path_map[clone_name] else: # no junction path found jpath = "Not Mounted"; # print results list += "%-25s %-30s %-20s " % (volume_name, snapshot, clone_name); list += "%11.2f MB " % parent_footprint; list += "%20.2f MB " % clone_footprint; # list += "%12.2f MB " % split_est; # list += "%19.2f MB" % clone_footprint; list += " %21.2f" % pctofparent; list += "%"; # list += " (%5.2f" % compression; print "%)"; # list += "%15s" % comment_field; list += " %s\n" % jpath; # get next set of records api = NaElement("volume-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Get next tag which indicates if there are more records tag = xo.child_get_string("next-tag") if (tag == None): break return list # # Prints out list of clones with their corresponding parent volume and parent snapshot def clist(self): # Get first set number of records defined by mrecs mrecs = "100" api = NaElement("volume-clone-get-iter") api.child_add_string("max-records", mrecs) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Set up to get next set of records tag = xo.child_get_string("next-tag") print("\n") # Need to go thru this loop at least once before checking for next tag while True: # Get list of snapshot-info attributes clonelist = dict() clonelist = xo.child_get("attributes-list") if (clonelist == None): break for clone in clonelist.children_get(): print ("clone %s:%s:%s" % (clone.child_get_string("parent-volume"), clone.child_get_string("parent-snapshot"), clone.child_get_string("volume"))) # Get next set of records api = NaElement("volume-clone-get-iter") api.child_add_string("max-records", mrecs) api.child_add_string("tag", tag) xo = self.get().invoke_elem(api) if(xo.results_status() == "failed"): raise NAException(xo.sprintf()) # Tag indicates if there are more records tag = xo.child_get_string("next-tag") # Break if no more records if (tag == None): break class NAException(Exception): def __init__(self, e): self.error = e # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Perforce connection as 'flex' user # --------------------------------------------------------------------------- class P4Flex: def __init__(self): self.port = config.get("p4.port") self.user = config.get("p4.user") self.passwd = config.get("p4.passwd") self.osuser = config.get("p4.osuser") def getP4(self): p4 = P4() p4.user = self.user p4.port = self.port p4.connect() p4.password = self.passwd p4.run_login() return p4 # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Parse the Broker's request and provide access to the user's environment # --------------------------------------------------------------------------- class Broker: def __init__(self): self.args = {} # Comment out stdin and uncomment block below for debug. lines = sys.stdin.readlines() for line in lines: parts = line.split(": ") self.args[parts[0]] = parts[1].rstrip() def getP4(self): p4 = P4() p4.user = self.args['user'] p4.port = self.args['clientPort'] p4.client = self.args['workspace'] p4.connect() # Use flex account to login user s4 = P4Flex().getP4() s4.run_login(self.args['user']) return p4 def getPort(self): return self.args['clientPort'] def getUser(self): return self.args['user'] def getClient(self): return self.args['workspace'] def getCommand(self): if 'Arg0' in self.args: return self.args['Arg0'] else: return None def getOptions(self): c = 1 opts = [] while True: key = 'Arg' + str(c) c += 1 if key in self.args: opts.append(self.args[key]) else: break join = "" list = [] for opt in opts: if join: opt = join + opt join = "" if opt.startswith('-') and len(opt) == 2: join = opt else: list.append(opt) if join: list.append(opt) return list # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Handle Broker request and invoke the registered 'flex' command # --------------------------------------------------------------------------- class Flex: def __init__(self): # Process Broker arguments self.call = Broker() self.command = self.call.getCommand() self.args = self.call.args self.opts = self.call.getOptions() # Build table for function pointers self.cmdFn = { 'volume': self.volume, 'volumes': self.volumes, 'lv': self.volumes, 'snapshot': self.snapshot, 'snapshots': self.snapshots, 'ls': self.snapshots, 'clone': self.clone, 'clones': self.clones, 'lc': self.clones, 'help': self.help }; if not self.isnfs(NaFlex().mount): print("action: REJECT") print("message: \"%s\"" % "NetApp nfs volume is not mounted.") return # Call command if self.command in self.cmdFn: self.cmdFn[self.command]() else: self.usage() def volume(self): # Check for 'admin' permission # NetApp connection netapp = NaFlex() if not self.permission(): print("action: REJECT") print("message: \"%s\"" % "You don't have permission for this operation.") return ontapversion = netapp.getontapversion() # Get options (set defaults) vname = "" size = "1G" user = self.call.getUser() uid = pwd.getpwnam(user).pw_uid gid = "" junction_path = "" for o in self.opts: if o.startswith('-d'): del_name = o[2:] self.vol_del(del_name) return if o.startswith('-s'): size = o[2:] if o.startswith('-j'): junction_path = o[2:] if o.startswith('-u'): uid = pwd.getpwnam(o[2:]).pw_uid if o.startswith('-g'): gid = grp.getgrnam(o[2:]).gr_gid else: vname = o if not gid: if uid: gid = pwd.getpwuid(uid).pw_gid else: p4group = config.get("p4.osuser") gid = pwd.getpwnam(p4group).pw_gid if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return if not junction_path: junction_path = "/" + vname # Create NetApp volume try: #with added cDOT 9 functionality netapp.create(vname, size,junction_path, uid, gid) # Set file ownership of newly created volume. user = self.call.getUser() path = netapp.mount + junction_path #Add in autogen of .p4config for new P4 Volume self.p4config(path, vname) # self.chown(user, path) msg = vname + ". Mounted on " + path + "." if "9." not in ontapversion: self.chown(user, path) msg += ("\n If necessary, run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path)) # msg += ("\n Run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path)) print("action: RESPOND") print("message: \"Created flex volume %s\"" % msg) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) except Exception as e: # If got unexpected error, delete the volume created error = "Unexpected Error: " + str(e) try: netapp.delete(vname) except NAException as e: error += '\nNetApp Error: ' + e.error print("action: RESPOND") print("message: \"%s\"" % error) def volumes(self): # NetApp connection netapp = NaFlex() # List the volumes try: list = "P4 Flex Volumes \n" vols = netapp.vlist() for v in vols: list += "volume '" list += v.child_get_string("name") + "' " list += "\n" print("action: RESPOND") print("message: \"%s\"" % list) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) # Builds a new FLEX_SNAP workspace to create the snapshot def snapshot(self): # Get options (set defaults) vname = "" sname = "" from_client_name = self.call.getClient() for o in self.opts: if o.startswith('-V'): vname = o[2:] if o.startswith('-d'): dname=o[2:] self.snap_del(vname,dname) return if o.startswith('-c'): from_client_name = o[2:] else: sname = o if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return if not sname: print("action: REJECT") print("message: \"%s\"" % "No flex clone name provided") return if ':' in sname: print("action: REJECT") print("message: \"%s\"" % "Flex clone name must not use ':'") return if not self.permission(): print("action: REJECT") print("message: \"%s\"" % "You don't have permission for this operation.") return # NetApp/Perforce connection as Flex netapp = NaFlex() p4 = P4Flex().getP4() # Create Perforce workspace for snapshot try: from_client = p4.fetch_client(from_client_name) root = from_client['Root'] # Clone client workspace flex_client_name = FLEX_SNAP + vname + ':' + sname p4.client = flex_client_name flex_client = p4.fetch_client("-t", from_client_name, flex_client_name) # Set workspace options: root to mounted volume path = netapp.mount + "/" + vname flex_client['Root'] = path flex_client['Host'] = "" #flex_client['Options'] = flex_client['Options'].replace(" unlocked", " locked") p4.save_client(flex_client) # Populate have list p4.run_sync("-k", "//" + flex_client_name + "/...@" + from_client_name) print("action: RESPOND") print("message: \"Created flex snapshot %s\"" % sname) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) sys.exit() finally: p4.disconnect() # NetApp/Perforce connection as Flex netapp = NaFlex() p4 = P4Flex().getP4() # Create NetApp snapshot try: netapp.snapshot(vname, sname) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) def snap_del(self, vname,snapname): if not snapname: print("action: REJECT") print("message: \"%s\"" % "No flex snapshot name provided") return if not vname: print("action: REJECT") print("message: \"%s\"" % "No volume name provided") return # NetApp/Perforce connection as Flex netapp = NaFlex() p4 = P4Flex().getP4() try: graboutput=str(netapp.snapshot_delete(vname,snapname)) if "ERROR" in graboutput: errstring = ("\n You must delete clones associcated with the snapshot %s first\n" %snapname) raise NAException(errstring + graboutput) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message: \"%s\"" % error) return #print ("\n NetApp Error : %s" %graboutput) # Delete Perforce workspaces try: client_name = FLEX_SNAP + vname + ":" + snapname p4.run_client("-f", "-d", client_name) print("action: RESPOND") message = "Deleted snapshot %s\n" % snapname print("message: \"%s\"" % message) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) finally: p4.disconnect() # If client delete succeed, Delete NetApp snapshot def permission(self): p4 = self.call.getP4() try: # Perforce connection as Caller # p4 = self.call.getP4() for p in p4.run_protects(): if (p['perm'] == 'admin') or (p['perm'] == 'super'): return True return False except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def snapshots(self): try: # Perforce connection as Caller p4 = self.call.getP4() snapshots = p4.run_clients("-e", FLEX_SNAP + "*") list = "" list += "%-25s %-30s %-45s \n" %("snapshot", "base-volume", "base-volume-path") for v in snapshots: name = v['client'][len(FLEX_SNAP):] part = name.split(':', 1) list += "%-25s %-30s %-45s \n" %(part[1], part[0], v['Root']) print("action: RESPOND") print("message: \"%s\"" % list) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() # Clones a new FLEX_CLONE workspace from the parent flexSnap_ def clone(self): # Get options vname = "" sname = "" cname = "" # Set directory ownership user = self.call.getUser() uid = pwd.getpwnam(user).pw_uid gid = "" junction_path = "" all=False for o in self.opts: if o.startswith('-d'): del_name = o[2:] self.clone_del(del_name) return if o.startswith('-V'): vname = o[2:] if o.startswith('-S'): sname = o[2:] if o.startswith('-j'): junction_path = o[2:] if o.startswith('-u'): uid = pwd.getpwnam(o[2:]).pw_uid if o.startswith('-g'): gid = grp.getgrnam(o[2:]).gr_gid else: cname = o if not gid: if uid: gid = pwd.getpwuid(uid).pw_gid else: p4group = config.get("p4.osuser") gid = pwd.getpwnam(p4group).pw_gid if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return if not sname: print("action: REJECT") print("message: \"%s\"" % "No flex snapshot name provided") return if not cname: print("action: REJECT") print("message: \"%s\"" % "No flex clone name provided") return if ':' in cname: print("action: REJECT") print("message: \"%s\"" % "Flex clone name must not use ':'") return if not junction_path: junction_path = netapp.mount + cname # NetApp/Perforce connection as Caller netapp = NaFlex() p4 = self.call.getP4() path = netapp.mount + "/" + cname # Create NetApp clone from snapshot ontapversion = netapp.getontapversion() # Generate the clone try: netapp.clone(cname, sname, vname, uid, gid,junction_path) except NAException as e: print("action: RESPOND") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) try: # Verify parent client exists parent_client_name = FLEX_SNAP + vname + ":" + sname list = p4.run_clients("-e", parent_client_name) if len(list) != 1: print("action: REJECT") print("message: \"Flex parent %s does not exist.\"" % parent_client_name) return # Clone client workspace clone_client_name = FLEX_CLONE + cname p4.client = clone_client_name clone_client = p4.fetch_client("-t", parent_client_name, clone_client_name) clone_client['Root'] = junction_path p4.save_client(clone_client) # Generate P4CONFIG file based on the clone path=netapp.mount+junction_path self.p4config(path, clone_client_name) # Populate have list p4.run_sync("-k", "//" + clone_client_name + "/...@" + parent_client_name) msg = clone_client_name + ". Mounted on " + junction_path + "." if "9." not in ontapversion: self.chown(user, path) msg += ("\n If necessary, run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path)) print("action: RESPOND") print("message: \"Created flex clone client %s\"" % msg) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def chown(self, user, path): # Checks need to be done in case of failure uid = pwd.getpwnam(user).pw_uid gid = pwd.getpwnam(user).pw_gid cmd=['sudo', 'chown', '-R', str(uid)+':'+str(gid), path] subprocess.call(cmd, shell=True) def isnfs(self, path): # Check if path is nfs mounted. fstype = subprocess.check_output('stat -f -L -c %T ' + path, shell=True) if (fstype == 'nfs\n'): return True return False def clones(self): # Get options all = False for o in self.opts: if o == "-a": all = True try: # Perforce connection as Caller p4 = self.call.getP4() clones = [] if all: clones = p4.run_clients("-e", FLEX_CLONE + "*") else: user = self.call.getUser() clones = p4.run_clients("-e", FLEX_CLONE + "*", "-u", user) list = "" if len(clones)!=0: netapp = NaFlex() message="" try: print("action: RESPOND") message += netapp.list_flexclones(clones) print("message: \"%s\"" % message) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) message += error print("error message: \"%s\"" % message) else: print("action: RESPOND") message = "No Clones Found" print("message: \"%s\"" % message) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def clone_del(self, cname): if not cname: print("action: REJECT") print("message: \"%s\"" % "No Flex clone name provided") return # NetApp/Perforce connection as Caller netapp = NaFlex() p4 = self.call.getP4() # Delete Perforce workspace try: client_name = FLEX_CLONE + cname p4.run_client("-d", client_name) print("action: REJECT") print("message: \"Deleted Flex clone %s\"" % cname) except P4Exception: print("action: REJECT") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) return finally: p4.disconnect() # If client delete succeed, Delete NetApp clone try: netapp.delete(cname) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message: \"%s\"" % error) return def vol_del(self, vname): graboutput="" if not vname: print("action: REJECT") print("message: \"%s\"" % "No flex volume name provided") return # NetApp/Perforce connection as Flex netapp = NaFlex() #First Check to see if any clones off the volume exist. If they do, exit and don't destroy anything try: if netapp.ckclone(vname)=="True": raise NAException ("\n The master volume %s has one or more clones which must be deleted first" %vname) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message:\"%s\"" % error) return try: graboutput=str(netapp.delete(vname)) if "error" in graboutput.lower(): errstring = ("\n You must delete clones associcated with the master volume %s first\n" %vname) raise NAException(errstring + graboutput) except NAException as e: print("action: REJECT") error = '\nNetApp Error: ' + e.error print("message: \"%s\"" % error) return except Exception as e: print("action: REJECT") error = '\nUnexpected error: ' + str(e) print("message:\"%s\"" % error) return # Delete Perforce workspaces # should only enter if upper part passed p4 = P4Flex().getP4() try: clones = p4.run_clients("-e", FLEX_SNAP + vname + "*") list = "" for c in clones: client_name = c['client'] p4.run_client("-f", "-d", client_name) list += " deleted client: %s\n" % client_name # print("action: RESPOND") message = "Deleted Flex volume %s\n" % vname message += list print("message: \"%s\"" % message) except P4Exception: print("action: RESPOND") error = '\n'.join(p4.errors) error += '\n'.join(p4.warnings) print("message: \"%s\"" % error) finally: p4.disconnect() def p4config(self, path, client): p4config = os.getenv('P4CONFIG', '.p4config') p4port = self.call.getPort() p4user = self.call.getUser() fh = open(path + "/.p4config", "wt") try: if fh: fh.write("# Generated by p4 flex.\n"); fh.write("P4PORT=" + p4port + "\n"); fh.write("P4USER=" + p4user + "\n"); fh.write("P4CLIENT=" + client + "\n"); finally: fh.close() fh.close() def help(self): help = ( "P4 FlexClone Usage Information\n\n" " P4 Broker functions for creating and managing NetApp FlexClones\n\n" "\n" "SYNOPSIS\n" " p4 flex [command] [command options]\n" "\n" "DESCRIPTION\n" " Create Volume\n" " p4 flex volume -s <vol size[M, G]> -j <junction_path> [-u user] [-g group] <volume name>\n" " Delete Volume\n" " p4 flex volume -d <volume name>\n" " List Volumes\n" " p4 flex volumes \n" " p4 flex lv\n" "\n" " Create Snapshot\n" " p4 flex snapshot -V <volume volume> [-c client] <snapshot name>\n" " Delete Snapshot\n" " p4 flex snapshot -V <volume name> -d <snapshot name>\n" " List Available Snapshots (with parent volume)\n" " p4 flex snapshots -V <volume name (optional)>\n" " p4 flex ls -V <volume name (optional)>\n" "\n" " Create FlexClone (aka clone)\n" " p4 flex clone -V <volume> -S <snapshot name> -j <junction_path> [-u user] [-g group] <clone name>\n" " Delete FlexClone\n" " p4 flex clone -d <clone name>\n" " List FlexClones\n" " p4 flex clones -V <volume name (optional)> [-a]\n" " p4 flex lc -V <volume name (optional)> [-a]\n" "\n" ) print("action: RESPOND") print("message: \"%s\"" % help) def usage(self): usage = ( "Usage: flex { volume(s) | snapshot(s) | clone(s) }\n" "Missing/wrong number of arguments." ) print("action: RESPOND") print("message: \"%s\"" % usage) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # MAIN # --------------------------------------------------------------------------- if __name__ == '__main__': flex = Flex()
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#9 | 24146 | Paul Allen | Copy changes from punithc/p4flex | ||
#8 | 20227 | netapp | small change in list_flexclones function to show the flexclone path in Linux, versus just the junction path. | ||
#7 | 20171 | netapp | flex.py has new code that uses volume footprints instead of old cDOT8 code when printing out the list of clones. | ||
#6 | 20103 | netapp |
Latest code - Aug 15, 2016. Changes made to address : 1. Added in functionality for snapshot delete ('snapshot -V volname -d snapshotsname') 2. Corrected volume delete to not allow for Helix database volume delete, unless all clones are first removed 3. 'p4 flex clones' now produces full table showing Parent Volume, Parent-Snapshot, FlexClone name, Parent volume space consumed, FlexClone volume space consumed, FlexClone acutal (space versus parent volume), and the Clone-path 4. Flex Clones now thinly provisioned to allow for storage space savings 5. Code change to process P4 first when taking snapshots, and then follow with NetApp step versus previous reverse order |
||
#5 | 19802 | netapp | Corrected empty clone list error. | ||
#4 | 19800 | netapp |
Many changes - 1 ) Snapshot Delete Capability 2) Sizing of FlexClones versus original volume reported showing space saved 3) Check for volume Clones before volume deletion 4) Output of chown command when volumes created by user |
||
#3 | 19636 | netapp | Reverting back to Agnes' original | ||
#2 | 19474 | netapp |
Uploaded new files for project. Enhancements include: 1. Added in snapshot deletion capability 2. Created separate user clone mount directory, which can be different from the base directory 3. Cleaned up error handling for all p4 flex calls, so errors are handled with understandable output, versus program crashes 4. Added in ability to specify volume owner when creating a volume with �p4 flex volume �s �u�. |
||
#1 | 18997 | netapp |
Rename/move file(s) Project name change: moving //guest/netapp/p4flexclone/main/… to //guest/netapp/p4flex/main/… |
||
//guest/netapp/p4flexclone/main/demo/flex.py | |||||
#36 | 15417 | Paul Allen | Remove client lock. | ||
#35 | 15070 | Paul Allen | spelling | ||
#34 | 15068 | Paul Allen | Allow locking on snapshot workspaces and use '-f' on volume delete. | ||
#33 | 15067 | Paul Allen |
Disable workspace locking for demo. Better solution is for user 'flex' to delete with a '-f', but check ownership first. |
||
#32 | 15066 | Paul Allen | Update create clone message. | ||
#31 | 15065 | Paul Allen | Back out changelist 15064 | ||
#30 | 15064 | Paul Allen | chown fix? | ||
#29 | 15063 | Paul Allen | Remove description from 'flex volumes' output. | ||
#28 | 15062 | Paul Allen |
Use Flex user to login current user (needs super) Minor fix to help |
||
#27 | 15027 | agnesj | Added a check if NetApp volume is mounted, removed test function, and modified -P to -S for flex clone command. | ||
#26 | 14926 | Paul Allen | Move chown operations to method and use system call to 'chown -R' | ||
#25 | 14925 | Paul Allen | Generate a P4CONFIG file in the mounted clone's volume. | ||
#24 | 14924 | Paul Allen | Set workspace 'Root' to mount location for clone and snapshot. | ||
#23 | 14923 | Paul Allen |
Show Owner when listing clones and volume when listing snapshots. Includes a tidy up on all listing commands. |
||
#22 | 14921 | Paul Allen |
#review-14917 Verify delete permission for volume/clone. - Only a Perforce 'admin' can delete a volume - Only a Perforce 'admin' and the clone's owner can delete a clone. Use Perforce client workspace based permissions to enforce rules. If the client delete succeeds assume all is ok to delete the volume/clone. Example error: bob$ p4 flex clone -d clone_2_1_a Locked client 'flexClone:clone_2_1_a' owned by 'pallen'; use -f to force update. |
||
#21 | 14895 | agnesj |
Modified volume function such that when creating a volume the owner of volume is modified. Also modified vol_del and clone_del functions to check for ownership of volume. Only owners of volumes are allowed to delete. |
||
#20 | 14881 | agnesj |
Added checks in chown and volume functions. And added location of newly created volume in message. |
||
#19 | 14180 | Paul Allen |
Set file ownership for clone's mounted files. + Minor fix to clone delete |
||
#18 | 14179 | Paul Allen | Add support for 'flex volumes' command. | ||
#17 | 14178 | agnesj | Modified NetApp delete to check if volume is a root volume (vserver or node) and to not allow p4 user to delete. | ||
#16 | 14141 | agnesj | Added definition functions for volume list, snapshot list based on volume name, all snapshot list, clone list | ||
#15 | 13997 | Paul Allen | Stop on error (don't delete clients if vol delete fails). | ||
#14 | 13996 | Paul Allen | Added -s to volume for size. | ||
#13 | 13993 | Paul Allen |
Added NetApp commands to Flex. Example: 4007::ws$ p4 flex volume vol1 Created flex volume vol1 4007::ws$ p4 flex snapshot -V vol1 snapshot1 Created flex snapshot snapshot1 4007::ws$ p4 flex snapshot -V vol1 snapshot2 Created flex snapshot snapshot2 4007::ws$ p4 flex snapshots snapshot vol1:snapshot1 root /home/pallen/flexclone/demo/ws snapshot vol1:snapshot2 root /home/pallen/flexclone/demo/ws 4007::ws$ p4 flex clone -V vol1 -P snapshot1 clone_A Created flex clone client flexClone:clone_A 4007::ws$ p4 flex clone -V vol1 -P snapshot1 clone_B Created flex clone client flexClone:clone_B 4007::ws$ p4 flex clones Clone clone_A root /home/pallen/flexclone/demo Clone clone_B root /home/pallen/flexclone/demo 4007::ws$ p4 clients Client flexClone:clone_A 2015/06/17 root /home/pallen/flexclone/demo 'Created by pallen. ' Client flexClone:clone_B 2015/06/17 root /home/pallen/flexclone/demo 'Created by pallen. ' Client flexSnap:vol1:snapshot1 2015/06/17 root /home/pallen/flexclone/demo/ws 'Created by flex. ' Client flexSnap:vol1:snapshot2 2015/06/17 root /home/pallen/flexclone/demo/ws 'Created by flex. ' Client pallen-ws 2015/06/10 root /home/pallen/flexclone/demo/ws 'Created by pallen. ' 4007::ws$ p4 flex clone -d clone_B Deleted Flex clone clone_B 4007::ws$ p4 flex clone -d clone_A Deleted Flex clone clone_A 4007::ws$ p4 flex volume -d vol1 Deleted Flex volume vol1 deleted client: flexSnap:vol1:snapshot1 deleted client: flexSnap:vol1:snapshot2 4007::ws$ p4 clients Client pallen-ws 2015/06/10 root /home/pallen/flexclone/demo/ws 'Created by pallen. ' |
||
#12 | 13988 | Paul Allen | Tidy up formatting and imports. | ||
#11 | 13985 | agnesj |
Added definitions: clone, snapshot, and delete Modified test functions to test each of the added functions |
||
#10 | 13867 | agnesj | Made corrections to NaFlex Class to pull info from flex.cfg file | ||
#9 | 13859 | Paul Allen | Config details for cloud demo + NaServer test | ||
#8 | 13796 | Paul Allen |
Moved options to a configuration file. - The config file 'flex.cfg' MUST be located in the same dir as the script - Provided a sample 'in.txt' file to simulate a broker call (used for debug only) |
||
#7 | 13791 | Paul Allen | Changed 'volume' to 'snapshot' and minor fix to for auth/login. | ||
#6 | 13702 | Paul Allen | Added Permission check for p4 flex volume command. | ||
#5 | 13671 | Paul Allen |
Added '-a' option to flex clones command. Tidy up and hide flexVol/flexClone prefix from user. |
||
#4 | 13669 | Paul Allen | Added '-d' option to delete flex clone client. | ||
#3 | 13668 | Paul Allen | Added Perforce functionality for volume(s) and clone(s) commands. | ||
#2 | 13628 | Paul Allen | set executable | ||
#1 | 13627 | Paul Allen | New Python script with Implemented help/usage |