manage_users_from_ldap.sh #9

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=off
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib" ||\
      { echo -e "\\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\\n"; exit 1; }
done

declare Version=1.4.5
declare -i SilentMode=0
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   vvmsg "$THISSCRIPT: EXITCODE: $ErrorCount"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Rotate log file and compress with gzip.
#------------------------------------------------------------------------------
rotate_log () {
   [[ "${P4U_LOG}" == off ]] && return 0
   declare Timestamp=
   declare RotatedLog=

   if [[ -f "$P4U_LOG" ]]; then
      Timestamp="$(date +'%Y-%m-%d_%H-%M-%S')"
      RotatedLog="${P4U_LOG}.${Timestamp}"

      mv -f "$P4U_LOG" "$RotatedLog" ||\
         bail "Failed to move log file aside with: mv -f  $P4U_LOG $RotatedLog"
      gzip "$RotatedLog" ||\
         warnmsg "Ignoring failure to gzip $RotatedLog."
   fi
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      errmsg "Bad Usage: $errorMessage"
   fi

   echo "USAGE for $THISSCRIPT v$Version:
 
$THISSCRIPT -i <instance> -lc <LDAPConfig> -ga <LDAPP4AdminUsersGroup> -gr <LDAPP4ReadUsersGroup> -gw <LDAPP4WriteUsersGroup> [-gA] [-c] [-d] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script provides a workflow based on various calls to the
	'p4 ldapsync' command, which support a methodolgy for
	establishing an LDAP server as the 'source of truth' for who
	should have access to a Helix Core server.

	This script can be called easily via cron.  In addition to
	using 'p4 ldapsync' to udpate groups from Perforce, this
	script adds a more comprehensive workflow for automatically
	adding and deleting users based on changes in LDAP. (Automatic
	deletion is strong discouraged, see EXAMPLES below).

	This scripts promtes a workflow with email notification for
	Helix Core admins, notifying about users to be added or removed,

	In some cases where 'p4 ldapsync -u -U' with '-u' does not work as
	expected, this script provides a workaround by creating missing
	users directly with 'p4 user' command.  This can happen if certain
	data elements in LDAP contain characters incompatible with Helix
	Core (e.g. '#' chars).

	This script wraps the 'p4 ldapsync' command, working with '-g'
	(group management) and '-u' user update modes.  It aborts the
	'p4 ldapsync' queries fails.

	The basic idea for this script is that all Helix Core users must be
	in one or more of these groups to have an account:
	* LDAPP4AdminUsersGroup
	* LDAPP4ReadUsersGroup
	* LDAPP4WriteUsersGroup
	* $NonLDAPP4UsersGroup

	Having Admin, Read, and Write group enables basic access controls
	if a Protections table is crafted to reference the different groups.
	If less granularity is required, the Admin, Read, and Write groups
	can be the same group name.  The actual granting of access occurs
	in the Protections table; this scripts responsibility is to update
	the Perforce groups based ln LDAP group membership.

	It is CRITICAL that all Helix Core users that are not in LDAP
	per local policy) be added to the group $NonLDAPP4UsersGroup.
	This typically includes automation users.  As an exception, user
	accounts that are only listed with the '-a' flag to 'p4 users',
	including accounts with a 'Type' value in the user spec of 'service'
	or 'operator', need not be listed. This script is, by design, unaware
	of such accounts, and will never attempt to delete them.  Such users
	always have a 'AuthMethod' value of 'perforce', and so are not going
	to be listed in LDAP.

	This script does the following:
	* Calls 'p4 ldapsync -g' to update the Perforce group $LDAPP4AdminUsersGroup
	from the LDAP group of the same name.

	* Calls 'p4 ldapsync -g' to update the Perforce group $LDAPP4ReadUsersGroup
	from the LDAP group of the same name.

	* Calls 'p4 ldapsync -g' to update the Perforce group $LDAPP4WriteUsersGroup
	from the LDAP group of the same name.

	* Detects users missing from Perforce, i.e. if an account is listed in
	any of the groups noted above, but does not have a Perforce account.
	Missing users will be reported, and optionally added with '-c'.

	* Detects extra users in Perforce, i.e. if any exist in Perforce but are
	not listed in any of the groups noted above.  Extra users will be
	reported.  If '-d' is specified, extra users will be deleted, along
	with all of their workspaces (according to the 'Owner' field of the
	client spec).  Workspace removal is done using a command like:

	p4 client -df -Fs <workspace_name>

	These flags will blast checkouts and deleted sheled files associated
	with the workspace.  To preserve shelved files, they must be unshelved
	by another user in another workspace prior to running this script.

	Workspace removal can fail in some edge cases, such as if the user has
	files checked out to a workspace for which they are not the listed
	Owner.  Manual corrective action is necessary in these cases.  If
	workspace removal fails, user removal will fail.

WORING AROUND A SPECIFIC LDAP INCOMPATIBILITY:
	This custom script is needed in scenarios where the command
	'p4 ldapsync' does not work in '-u' (update) mode.  This can
	happen if certain data elements in LDAP contain characters
	incompatible with Perforce (e.g. '#' chars).

OPTIONS:
 -i <instance>
	Specify the SDP instance name.  If the '-i' flag is omitted, the value
	is derived from the \$SDP_INSTANCE environment variable.

	If \$SDP_INSTANCE is not defined, then '-i <instance>' is required.

 -lc	Specify the LDAP config.  This is required.

 -gA	Indicate that all Perforce groups with LDAP config settings are to
	be updated.  This can be useful if the Protections table is crafted
	to reference multiple Helix Core groups connected to LDAP groups.

	Only groups groups specified with required -ga, -gr, and -gw determine
	which accounts are created/deleted, or suggested for creation/removal,
	depending on usage of '-c' and '-d' flags.

 -ga	Specify the LDAP group for admin users. Required.

 -gr	Specify the LDAP group for read-only users. Required.

 -gw	Specify the LDAP group for write  users. Required.

 -c	Create users that exist in either of the two user groups mentioned
	above but which do not exist in Perforce.

	Users will be added with an Email field value of <userid>@<domain>,
	where domain is determined from the MAILFROM setting, which evaluates
	to @$EmailDomain.  The FullName field is set to the same value as
	the userid, and can be adjusted by the user manually.  The AuthMethod
	will be set to whatever the default is (per the auth.default.method
	configurable).

	By default, without '-c', users to be added are reported, but no
	action is taken.

	As a safety feature, a maximum of $MaxUsersToAdd missing users will
	be added on any one invocation of this script.  To add more users,
	call this script as many times as needed.

 -d	Delete extra uses and any client specs (workspaces) for which they
	are the listed Owner.  By default, without '-d', users to be added
	are reported, but no action is taken.

	As a safety feature, a maximum of $MaxUsersToDelete extra users will
	be deleted on any one invocation of this script.  To delete more users,
	call this script as many times as needed.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

	NOTE: This script is self-logging.  That is, output displayed on the
	screen is simultaneously captured in the log file.  Do not run this
	script with redirection operators like '> log' or '2>&1', and do not
	use 'tee'.

-si	Operate silently.  All output (stdout and stderr) is redirected to the
	log only; no output appears on the terminal.
	
 -n	No-Op.  Prints certain commands instead of running them.  Some
	commands, such as the 'p4 ldapsync -g' command that does not affect
	data, are executd regardless of whether '-n' is used.  Using '-n'
	will prevent creation and removal of users even if '-c' and/or '-d'
	are used.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

EXAMPLES:
	The following is a sample call from 'cron', typically once per
	day:

	manage_users_from_ldap.sh -i 1 -gA -ga P4Admins -gw P4Write -gr P4Read -c
	In this example, the P4Admins, P4Write, and P4Read groups would be
	Perforce groups with 'LdapConfig' and other AD connection settings
	defined.

	This sample illustrates the preferred policy:  Use '-c' to
	automatically create users from AD, but DO NOT use '-d' to
	automatically delete users.

	Using '-c' is recommended as there is only benefit to having Perforce
	user accounts created automtically once they have been added to
	appropriate groups in LDAP indicating they are authorized to have
	Perforce Helix Core access.

	Using '-d' is STRONGLY DISCOURAGED, as it risks data loss of work-in-
	progress data from recently departed staff.  Instead, human admins
	should review the list of users suggested for removal, and delete
	them with manual processes, perhaps with coordination with managers
	or colleagues of departed staff to ensure no useful data is lost.
	For example, staff may want to review lists of workspaces and checked
	out files to see if anything valuable is left.  This sort of analysis
	is most needed in cases of sudden and/or unplanned staff departure.

	Risks of using -d include, but are not limited to:
	* Sometimes workspaces are shared by multiple users.  This script
	deletes all workspaces for which the to-be-deleted user is listed as
	the Owner based on the client spec.  Manual reivew of workspaces to
	be deleted (which this script provides when '-d' is not provided)
	gives a trail to follow to look for potential scenarios like this.
	If the departed user was helpful and helped other users create
	workspaces or created build workspaces, changing the Owner field
	is required to prevent their removal.
	* Checked out files of departed users will be forgotten about.
	While the actual contents of such files on client machines is not
	affected, the server's knowedge of those being checked out is lost
	when the user workspace is deleted on the server.
	* Shelved files will be deleted.  If they are of value, they should
	first be transferred to another user.
	* If admins make mistakes in putting users in the correct LDAP groups,
	it could cause accidental removal of active human or robot accounts
	(which might not be on LDAP).

	All this said, there are scenarios for which the '-d' flag is useful,
	and so it exists.

LDAP CONFIG:
	This script replies on the LDAP configuration working properly. This
	implies that an LDAP 'reader' account is configured in an LDAP spec,
	and that the required LDAP credentials are codified (encrypted) in
	the 'p4 ldap' spec.

	See:
	* https://www.perforce.com/manuals/p4sag/Content/P4SAG/security.ldap.auth.html
	* p4 help ldap, for ldap specs
	* p4 help group, for per-group LDAP configuration

	Comments: When crafting your LDAP configs and LDAP group configs,
	it is wise to put effort into optimizing granular seach queries,
	to optimize performance and reduce load on the LDAP server. This
	can also prevent obscure timeout failures and other scale-related
	errors.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

export SDP_INSTANCE="${SDP_INSTANCE:-Unset}"
declare LDAPConfig="Unset"
declare LDAPP4AdminUsersGroup="Unset"
declare LDAPP4ReadUsersGroup="Unset"
declare LDAPP4WriteUsersGroup="Unset"
declare NonLDAPP4UsersGroup="Non-LDAP-Users"
declare DefaultMailFrom=${MAILFROM:-P4Admin@p4demo.com}
declare EmailDomain=${DefaultMailFrom##*@}
declare TmpFile=
declare -i SyncAllLDAPGroups=0
declare -i KeepLogs=7
declare -i CreateMissingUsers=0
declare -i DeleteExtraUsers=0
declare -i MaxUsersToAdd=5
declare -i MaxUsersToDelete=5
declare -i UsersToAddCount=0
declare -i UsersToDeleteCount=0
declare -i ClientsToDeleteCount=0
declare -i UserAddErrorCount=0
declare -i UserDeleteErrorCount=0
declare -i ClientDeleteErrorCount=0
declare -i ErrorCount=0
declare -A P4Users
declare -A LDAPUsers
declare -a UsersToAdd
declare -a UsersToDelete
declare -i i=0

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) SDP_INSTANCE=$2; shiftArgs=1;;
      (-c) CreateMissingUsers=1;;
      (-lc) LDAPConfig=$2; shiftArgs=1;;
      (-gA) SyncAllLDAPGroups=1;;
      (-ga) LDAPP4AdminUsersGroup=$2; shiftArgs=1;;
      (-gr) LDAPP4ReadUsersGroup=$2; shiftArgs=1;;
      (-gw) LDAPP4WriteUsersGroup=$2; shiftArgs=1;;
      (-d) DeleteExtraUsers=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SDP_INSTANCE" == Unset ]] && \
   bail "The '-i <instance>' is required unless \$SDP_INSTANCE is defined."

[[ "$LDAPConfig" == Unset ]] && \
   bail "The '-lc <LDAPConfig>' argument is required."

[[ "$LDAPP4AdminUsersGroup" == Unset ]] && \
   bail "The '-ga <LDAPP4AdminUsersGroup>' argument is required."

[[ "$LDAPP4ReadUsersGroup" == Unset ]] && \
   bail "The '-gr <LDAPP4ReadUsersGroup>' argument is required."

[[ "$LDAPP4WriteUsersGroup" == Unset ]] && \
   bail "The '-gw <LDAPP4WriteUsersGroup>' argument is required."

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDP_INSTANCE"

# Set variables that depend on SDP environment settings.
export P4U_LOG="${LOGS}/manage_users_from_ldap.log"
EmailDomain=${MAILFROM##*@}
TmpFile=${P4TMP}/tmp.$$.$RANDOM

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   rotate_log

   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

msg "\\nStarted ${0##*/} v$Version.\\n"

if [[ "$SyncAllLDAPGroups" -eq 1 ]]; then
   run "$P4BIN -s ldapsync -g" \
      "Updating all Perforce groups connected to LDAP." 1 1 ||\
      bail "Failed to update all groups from LDAP."
else
   run "$P4BIN -s ldapsync -g $LDAPP4AdminUsersGroup" \
      "Updating Perforce group $LDAPP4AdminUsersGroup from LDAP group of the same name." 1 1 ||\
      bail "Failed to update group from LDAP."

   run "$P4BIN -s ldapsync -g $LDAPP4ReadUsersGroup" \
      "Updating Perforce group $LDAPP4ReadUsersGroup from LDAP group of the same name." 1 1 ||\
      bail "Failed to update group from LDAP."

   run "$P4BIN -s ldapsync -g $LDAPP4WriteUsersGroup" \
      "Updating Perforce group $LDAPP4WriteUsersGroup from LDAP group of the same name." 1 1 ||\
      bail "Failed to update group from LDAP."
fi

run "$P4BIN ldapsync -u -U $LDAPConfig" \
   "Updating Perforce user data from LDAP." 1 1 ||\
   bail "Failed to update user data from LDAP."

for user in $("$P4BIN" -ztag group -o "$LDAPP4AdminUsersGroup" | grep '^\.\.\. Users' | awk '{print $3}'); do
   LDAPUsers[$user]="$user"
done

for user in $("$P4BIN" -ztag group -o "$LDAPP4ReadUsersGroup" | grep '^\.\.\. Users' | awk '{print $3}'); do
   LDAPUsers[$user]="$user"
done

for user in $("$P4BIN" -ztag group -o "$LDAPP4WriteUsersGroup" | grep '^\.\.\. Users' | awk '{print $3}'); do
   LDAPUsers[$user]="$user"
done

for user in $("$P4BIN" -ztag group -o "$NonLDAPP4UsersGroup" | grep '^\.\.\. Users' | awk '{print $3}'); do
   LDAPUsers[$user]="$user"
done

# Note: We intentionally use 'p4 users' without '-a', as we want to exclude
# service accounts, which must have an AuthMethod of 'perforce'.  So by
# design, this script is unaware of 'service' and 'operator' accounts and
# will never propopose to delete them.
for user in $("$P4BIN" -ztag -F %User% users); do
   P4Users[$user]="$user"
done

msg "${H2}\\nFinding LDAP users to add to Perforce."
i=0
for user in "${LDAPUsers[@]}"; do
   if [[ -z "${P4Users[$user]:-}" ]]; then
      UsersToAdd[$i]="$user"
      i+=1
   fi
done

if [[ "$i" -eq 0 ]]; then
   msg "No users need to be added."
else
   msg "Found $i users missing from Perforce."
   if [[ "$CreateMissingUsers" -eq 1 ]]; then
      if [[ "$i" -le "$MaxUsersToAdd" ]]; then
         msg "Attempting to add all $i missing users."
      else
         msg "Attempting to add $MaxUsersToAdd of the $i missing users."
      fi
   fi

   for user in "${UsersToAdd[@]}"; do
      UsersToAddCount+=1

      if [[ "$CreateMissingUsers" -eq 1 && "$UsersToAddCount" -gt "$MaxUsersToAdd" ]]; then
         warnmsg "Created max of $MaxUsersToAdd. Only reporting remaining users to add."
         CreateMissingUsers=0
      fi

      echo -e "User: $user\\n\\nEmail: ${user}@${EmailDomain}\\n\\nFullName: $user\\n\\nAuthMethod: ldap\\n\\n" > "$TmpFile" ||\
         bail "Failed to write to temp file: $TmpFile\\n"

      if [[ "$CreateMissingUsers" -eq 1 ]]; then
         run "$P4BIN -s user -f -i < $TmpFile" \
            "Creating user with this generated user spec:\\n$(cat "$TmpFile")\\n" \
            1 1 ' saved'

         if [[ "$CMDEXITCODE" -ne 0 ]]; then
            errmsg "Failed to add user: $user\\n"
            UserAddErrorCount+=1
            ErrorCount+=1
         fi
      else
         msg "Add user: $user"
      fi
   done
fi

msg "${H2}\\nFinding extra Perforce users to remove that do not exist in LDAP or the group $NonLDAPP4UsersGroup."
i=0
for user in "${P4Users[@]}"; do
   if [[ -z "${LDAPUsers[$user]:-}" ]]; then
      UsersToDelete[$i]="$user"
      i+=1
   fi
done

if [[ "$i" -eq 0 ]]; then
   msg "No users need to be removed."
else
   msg "Found $i extra users in Perforce."

   if [[ "$DeleteExtraUsers" -eq 1 ]]; then
      if [[ "$i" -le "$MaxUsersToDelete" ]]; then
         msg "Attempting to delete all $i extra users."
      else
         msg "Attempting to delete $MaxUsersToDelete of the $i extra users."
      fi
   fi

   for user in "${UsersToDelete[@]}"; do
      msg "Delete user: $user"
      UsersToDeleteCount+=1

      if [[ "$DeleteExtraUsers" -eq 1 && "$UsersToDeleteCount" -gt "$MaxUsersToDelete" ]]; then
         warnmsg "Deleted max of $MaxUsersToDelete. Only reporting remaining users to delete."
         DeleteExtraUsers=0
      fi

      for client in $("$P4BIN" -ztag -F %client% clients -u "$user"); do
         ClientsToDeleteCount+=1

         if [[ "$DeleteExtraUsers" -eq 1 ]]; then
            run "$P4BIN -s client -df -Fs $client" \
               "\\n\\tDelete client: $client" 1 1 ' deleted'

            if [[ "$CMDEXITCODE" -ne 0 ]]; then
               errmsg "Failed to remove client: $client\\n"
               ClientDeleteErrorCount+=1
               ErrorCount+=1
            fi
         else
            msg "\\n\\tDelete client: $client"
         fi
      done

      if [[ "$DeleteExtraUsers" -eq 1 ]]; then
         run "$P4BIN -s user -d -f $user" \
            "Delete user: $user" 1 1 ' deleted'

         if [[ "$CMDEXITCODE" -ne 0 ]]; then
            errmsg "Failed to remove user: $user\\n"
            UserDeleteErrorCount+=1
            ErrorCount+=1
         fi
      else
         msg "\\n\\tDelete user: $user"
      fi
   done
fi

msg "${H2}\\nSummary:\\n\\tUsers to add: $UsersToAddCount\\n\\tUsers to remove: $UsersToDeleteCount\\n\\tClients to remove: $ClientsToDeleteCount\\n\\tUser Add Errors: $UserAddErrorCount\\n\\tUser Delete Errors: $UserDeleteErrorCount\\n\\tClient Delete Errors: $ClientDeleteErrorCount\\n\\tTotal Errors: $ErrorCount\\n\\n"

if [[ $ErrorCount -eq 0 ]]; then
   msg "${H}\\nAll processing completed successfully.\\n"
   msg "Cleaning up logs older than $KeepLogs days old."

  find "$LOGS/" -name "manage_users_from_ldap.log.*" -mtime +$KeepLogs -print -exec /bin/rm -f {} \;

else
   msg "${H}\\nProcessing completed, but with $ErrorCount errors detected.  Scan above output carefully.\\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit $ErrorCount