#!/usr/bin/env python3
# -*- encoding: UTF8 -*-
# Test harness for JobsCmdFilter.py
from __future__ import print_function
import sys
import unittest
import os
import io
import logging
import re
import sqlite3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from log2sql import Log2sql
python3 = sys.version_info[0] >= 3
if python3:
from io import StringIO
else:
from StringIO import StringIO
fileHeader = """CREATE DATABASE IF NOT EXISTS testdb;
USE testdb;
DROP TABLE IF EXISTS process;
CREATE TABLE process (processkey CHAR(50) NOT NULL, lineNumber INT NOT NULL, pid INT NOT NULL, startTime DATETIME NOT NULL,endTime DATETIME NULL, computedLapse FLOAT NULL,completedLapse FLOAT NULL, user TEXT NOT NULL, workspace TEXT NOT NULL, ip TEXT NOT NULL, app TEXT NOT NULL, cmd TEXT NOT NULL, args TEXT NULL, uCpu INT NULL, sCpu INT NULL, diskIn INT NULL, diskOut INT NULL, ipcIn INT NULL, ipcOut INT NULL, maxRss INT NULL, pageFaults INT NULL, rpcMsgsIn INT NULL, rpcMsgsOut INT NULL, rpcSizeIn INT NULL, rpcSizeOut INT NULL, rpcHimarkFwd INT NULL, rpcHimarkRev INT NULL, rpcSnd FLOAT NULL, rpcRcv FLOAT NULL, running INT NULL, error TEXT NULL, PRIMARY KEY (processkey, lineNumber));
DROP TABLE IF EXISTS tableUse;
CREATE TABLE tableUse (processkey CHAR(50) NOT NULL, lineNumber INT NOT NULL, tableName VARCHAR(255) NOT NULL, pagesIn INT NULL, pagesOut INT NULL, pagesCached INT NULL, pagesSplitInternal INT NULL, pagesSplitLeaf INT NULL, readLocks INT NULL, writeLocks INT NULL, getRows INT NULL, posRows INT NULL, scanRows INT NULL, putRows int NULL, delRows INT NULL, totalReadWait INT NULL, totalReadHeld INT NULL, totalWriteWait INT NULL, totalWriteHeld INT NULL, maxReadWait INT NULL, maxReadHeld INT NULL, maxWriteWait INT NULL, maxWriteHeld INT NULL, peekCount INT NULL, totalPeekWait INT NULL, totalPeekHeld INT NULL, maxPeekWait INT NULL, maxPeekHeld INT NULL, triggerLapse FLOAT NULL, PRIMARY KEY (processkey, lineNumber, tableName));
BEGIN TRANSACTION;
"""
def compressGzip(tmpFile, input):
import gzip
with gzip.open(tmpFile, 'wb') as f:
if python3:
f.write(bytes(input.encode()))
else:
f.write(input)
def compressZip(tmpFile, input):
from zipfile import ZipFile
tmpContentsFile = "~tmpContents.log"
with open(tmpContentsFile, "w") as f:
f.write(input)
with ZipFile(tmpFile, 'w') as myzip:
myzip.write(tmpContentsFile)
class MyOptions():
def __init__(self, dbname, logname, sql=False, verbosity=logging.DEBUG):
self.dbname = dbname
self.logname = logname
self.logfile = [logname]
self.sql = sql
self.no_sql = not sql
self.verbosity = verbosity
self.interval = 10
self.outlog = None
self.reset = False
class TestLogParser(unittest.TestCase):
def __init__(self, methodName='runTest'):
super(TestLogParser, self).__init__(methodName=methodName)
self.maxDiff = None
def assertRegex(self, *args, **kwargs):
if python3:
return super(TestLogParser, self).assertRegex(*args, **kwargs)
else:
return super(TestLogParser, self).assertRegexpMatches(*args, **kwargs)
def setUp(self):
pass
def tearDown(self):
pass
def runTestText(self, expected, input, longlines=False, compressed=False):
if not compressed:
optionsText = MyOptions("testdb", "")
else:
optionsText = MyOptions("testdb", input)
self.saved_stdoutput = StringIO()
self.saved_stderr = StringIO()
if not compressed:
log2sql = Log2sql(optionsText, instream=StringIO(input), outstream=self.saved_stdoutput, errstream=self.saved_stderr)
else:
optionsText.logname = input
log2sql = Log2sql(optionsText, outstream=self.saved_stdoutput, errstream=self.saved_stderr)
log2sql.processLogs()
actual = self.saved_stdoutput.getvalue()
self.assertMultiLineEqual(fileHeader, actual[:len(fileHeader)])
if not longlines:
exp = "\n".join(sorted(expected.split("\n")))
act = "\n".join(sorted(actual[len(fileHeader):].split("\n")))
self.assertMultiLineEqual(exp, act)
else:
rest = actual[len(fileHeader):]
for expline in expected.split("\n"):
reLine = re.compile(expline.replace("(", "\("), re.MULTILINE)
self.assertRegex(rest, reLine, re.MULTILINE)
reUnrecognised = re.compile("Unrecognised track:", re.MULTILINE)
if re.search(reUnrecognised, self.saved_stderr.getvalue()):
self.assertTrue(False, "Didn't process all track records: %s" % self.saved_stderr.getvalue())
def runTestSQL(self, input, compressed=False):
if not compressed:
optionsSQL = MyOptions("testdb", "", sql=True)
else:
optionsSQL = MyOptions("testdb", input, sql=True)
dbname = "%s.db" % optionsSQL.dbname
if os.path.exists(dbname):
os.remove(dbname)
saved_stdoutput = StringIO()
saved_stderr = StringIO()
if not compressed:
log2sql = Log2sql(optionsSQL, instream=StringIO(input), outstream=saved_stdoutput, errstream=saved_stderr)
else:
optionsSQL.logname = input
log2sql = Log2sql(optionsSQL, outstream=saved_stdoutput, errstream=saved_stderr)
log2sql.processLogs()
return sqlite3.connect(dbname)
def testClientLockRecords(self):
"Extra record for client lock"
# Test parsing of sync with client lock info
input = """
Perforce server info:
2017/02/15 13:46:42 pid 81805 bruno@robert_cowham-dvcs-1487082773 10.62.185.98 [p4/2016.2/LINUX26X86_64/1468155] 'user-client -d -f bruno.139631598948304.irp210-h03'
Perforce server info:
2017/02/15 13:46:42 pid 81805 bruno@robert_cowham-dvcs-1487082773 10.62.185.98 [p4/2016.2/LINUX26X86_64/1468155] 'user-client -d -f bruno.139631598948304.irp210-h03'
--- clients/bruno%2E139631598948304%2Eirp210-h03(W)
--- total lock wait+held read/write 0ms+0ms/0ms+9ms
Perforce server info:
2017/02/15 13:46:42 pid 81805 completed .009s 8+1us 0+1408io 0+0net 4088k 0pf
Perforce server info:
2017/02/15 13:46:42 pid 81805 bruno@robert_cowham-dvcs-1487082773 10.62.185.98 [p4/2016.2/LINUX26X86_64/1468155] 'user-client -d -f bruno.139631598948304.irp210-h03'
--- lapse .009s
--- usage 10+11us 12+13io 14+15net 4088k 0pf
--- rpc msgs/size in+out 20+21/22mb+23mb himarks 318788/318789 snd/rcv .001s/.002s
--- db.have
--- pages in+out+cached 1+2+3
--- pages split internal+leaf 41+42
--- locks read/write 4/5 rows get+pos+scan put+del 6+7+8 9+10
--- total lock wait+held read/write 12ms+13ms/14ms+15ms
--- max lock wait+held read/write 32ms+33ms/34ms+35ms
--- peek count 20 wait+held total/max 21ms+22ms/23ms+24ms
"""
md5 = "7868f2723d35c6cb91784afa6bef4a7a"
expected = """INSERT IGNORE INTO tableUse VALUES ("%s",2,"have",1,2,3,41,42,4,5,6,7,8,9,10,12,13,14,15,32,33,34,35,20,21,22,23,24,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"clients_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,9,0,0,0,9,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("%s",2,81805,"2017-02-15 13:46:42","2017-02-15 13:46:42",NULL,0.009,"bruno","robert_cowham-dvcs-1487082773","10.62.185.98","p4/2016.2/LINUX26X86_64/1468155","user-client","-d -f bruno.139631598948304.irp210-h03",10,11,12,13,14,15,4088,0,20,21,22,23,318788,318789,.001,.002,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT processkey, lineNumber, pid, startTime, endTime, computedLapse, completedLapse, '
'user, workspace, ip, app, cmd, args, uCpu, sCpu, diskIn, diskOut, ipcIn, ipcOut, maxRss, pageFaults, '
'rpcMsgsIn, rpcMsgsOut, rpcSizeIn, rpcSizeOut, rpcHimarkFwd, rpcHimarkRev, rpcSnd, rpcRcv, running from Process;')
rows = c.fetchall()
self.assertEqual(1, len(rows))
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[1], 2) # lineNumber
self.assertEqual(row[2], 81805) # pid
self.assertEqual(row[3], '2017-02-15 13:46:42') # startTime
self.assertEqual(row[4], '2017-02-15 13:46:42') # endTime
self.assertEqual(row[5], None) # computedLapse
self.assertEqual(row[6], 0.009) # completedLapse
self.assertEqual(row[7], "bruno") # user
self.assertEqual(row[8], "robert_cowham-dvcs-1487082773") # workspace
self.assertEqual(row[9], "10.62.185.98") # ip
self.assertEqual(row[10], "p4/2016.2/LINUX26X86_64/1468155") # app
self.assertEqual(row[11], "user-client") # cmd
self.assertEqual(row[12], "-d -f bruno.139631598948304.irp210-h03") # args
self.assertEqual(row[13], 10) # uCpu
self.assertEqual(row[14], 11) # sCpu
self.assertEqual(row[15], 12) # diskIn
self.assertEqual(row[16], 13) # diskOut
self.assertEqual(row[17], 14) # ipcIn
self.assertEqual(row[18], 15) # ipcOut
self.assertEqual(row[19], 4088) # maxRss
self.assertEqual(row[20], 0) # pageFaults
self.assertEqual(row[21], 20) # rpcMsgsIn
self.assertEqual(row[22], 21) # rpcMsgsOut
self.assertEqual(row[23], 22) # rpcSizeIn
self.assertEqual(row[24], 23) # rpcSizeOut
self.assertEqual(row[25], 318788) # rpcHimarkFwd
self.assertEqual(row[26], 318789) # rpcHimarkRev
self.assertEqual(row[27], 0.001) # rpcSnd
self.assertEqual(row[28], 0.002) # rpcRcv
self.assertEqual(row[29], 0) # running
c.execute('SELECT processkey, lineNumber, tableName, pagesIn, pagesOut, pagesCached, '
'pagesSplitInternal, pagesSplitLeaf, readLocks, writeLocks, '
'getRows, posRows, scanRows, putRows, delRows, totalReadWait, totalReadHeld, totalWriteWait, '
'totalWriteHeld, maxReadWait, maxReadHeld, maxWriteWait, maxWriteHeld, '
'peekCount, totalPeekWait, totalPeekHeld, maxPeekWait, maxPeekHeld, triggerLapse from tableUse;')
rows = c.fetchall()
self.assertEqual(2, len(rows))
row = rows[0] # Ignore client row
self.assertEqual(row[0], md5)
self.assertEqual(row[1], 2) # lineNumber
self.assertEqual(row[2], 'have') # tableName
self.assertEqual(row[3], 1) # pagesIn
self.assertEqual(row[4], 2) # pagesOut
self.assertEqual(row[5], 3) # pagesCached
self.assertEqual(row[6], 41) # pagesSplitInternal
self.assertEqual(row[7], 42) # pagesSplitLeaf
self.assertEqual(row[8], 4) # readLocks
self.assertEqual(row[9], 5) # writeLocks
self.assertEqual(row[10], 6) # getRows
self.assertEqual(row[11], 7) # posRows
self.assertEqual(row[12], 8) # scanRows
self.assertEqual(row[13], 9) # putRows
self.assertEqual(row[14], 10) # delRows
self.assertEqual(row[15], 12) # totalReadWait
self.assertEqual(row[16], 13) # totalReadHeld
self.assertEqual(row[17], 14) # totalWriteWait
self.assertEqual(row[18], 15) # totalWriteHeld
self.assertEqual(row[19], 32) # maxReadWait
self.assertEqual(row[20], 33) # maxReadHeld
self.assertEqual(row[21], 34) # maxWriteWait
self.assertEqual(row[22], 35) # maxWriteHeld
self.assertEqual(row[23], 20) # peekCount
self.assertEqual(row[24], 21) # totalPeekWait
self.assertEqual(row[25], 22) # totalPeekHeld
self.assertEqual(row[26], 23) # maxPeekWait
self.assertEqual(row[27], 24) # maxPeekHeld
conn.close()
def testBasicParsing(self):
"simple parsing"
# Test parsing of sync with 3 log entries
input = """
Perforce server info:
2015/09/02 15:23:09 pid 1616 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-sync //...'
Perforce server info:
2015/09/02 15:23:09 pid 1616 compute end .031s
Perforce server info:
2015/09/02 15:23:09 pid 1616 completed .031s
"""
md5 = "4d4e5096f7b732e4ce95230ef085bf51"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,1616,"2015-09-02 15:23:09","2015-09-02 15:23:09",0.031,0.031,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-sync","//...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2015-09-02 15:23:09')
conn.close()
# Same as above with invalid Unicode strings
input = """
Perforce server info:
2015/09/02 15:23:09 pid 1616 robert@robert-test 127.0.0.1 [Microsoft® Visual Studio® 2013/12.0.21005.1] 'user-sync //...'
Perforce server info:
2015/09/02 15:23:09 pid 1616 compute end .031s
Perforce server info:
2015/09/02 15:23:09 pid 1616 completed .031s
"""
md5 = "e5a1bef75cf1652dcee6a4a7fe714cd4"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,1616,"2015-09-02 15:23:09","2015-09-02 15:23:09",0.031,0.031,"robert","robert-test","127.0.0.1","Microsoft® Visual Studio® 2013/12.0.21005.1","user-sync","//...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
if python3:
md5 = "003494353720ac68a0ddbbf1fd17288f"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,1616,"2015-09-02 15:23:09","2015-09-02 15:23:09",0.031,0.031,"robert","robert-test","127.0.0.1","Microsoft® Visual Studio® 2013/12.0.21005.1","user-sync","//...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
# self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2015-09-02 15:23:09')
conn.close()
def runCompressedTest(self, compressFunction):
"simple test of compressed input"
# Test parsing of sync with 3 log entries
input = """
Perforce server info:
2015/09/02 15:23:09 pid 1616 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-sync //...'
Perforce server info:
2015/09/02 15:23:09 pid 1616 compute end .031s
Perforce server info:
2015/09/02 15:23:09 pid 1616 completed .031s
"""
md5 = "4d4e5096f7b732e4ce95230ef085bf51"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,1616,"2015-09-02 15:23:09","2015-09-02 15:23:09",0.031,0.031,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-sync","//...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
tmpFile = "~tmp.log.compressed"
compressFunction(tmpFile, input)
self.runTestText(expected, tmpFile, compressed=True)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2015-09-02 15:23:09')
conn.close()
def testGzip(self):
"simple test of compressed input"
self.runCompressedTest(compressFunction=compressGzip)
def testZip(self):
"simple test of compressed input"
self.runCompressedTest(compressFunction=compressZip)
def testThreeRecords(self):
"Extra record for client lock"
# Test parsing of sync with 3 log entries
input = """
Perforce server info:
2015/09/02 15:23:11 pid 1616 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-changes -c robert-test -m 1 -s pending -u robert'
Perforce server info:
2015/09/02 15:23:11 pid 1616 compute end .047s
Perforce server info:
2015/09/02 15:23:11 pid 1616 completed .047s
"""
md5 = "2e9997f8255ea05d99647ad0f81f793e"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,1616,"2015-09-02 15:23:11","2015-09-02 15:23:11",0.047,0.047,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-changes","-c robert-test -m 1 -s pending -u robert",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2015-09-02 15:23:11')
self.assertEqual(row[4], '2015-09-02 15:23:11')
self.assertEqual(row[5], 0.047)
conn.close()
def testSwarm(self):
"Swarm style entry"
# Test parsing of swarm style entry
input = r"""
Perforce server info:
2016/12/21 08:39:39 pid 14769 perforce@~tmp.1482305462.13038.585a2fb6041cc1.60954329 192.168.18.31 [SWARM/2016.2/1446446] 'user-counter -u swarm-activity-fffec3dd {"type":"change","link":["change",{"change":1005814}],"user":"sahaltran05","action":"committed","target":"change 1005814","preposition":"into","description":"Mac address filtering and fixing the naming collision for the SSH and telnet libraries\n","details":null,"topic":"changes\/1005814","depotFile":null,"time":1482305978,"behalfOf":null,"projects":{"sah-automation":["sah-tests"]},"streams":["user-sahaltran05","personal-sahaltran05","project-sah-automation","group-p4altran","group-sah_app","group-sah_commun_modules","group-sah_components","group-sah_demo","group-sah_hardco","group-sah_nanterre","group-sah_nanterre_opensource","group-sah_opensource","group-sah_stbconfig","group-sah_stbconfig_dev","group-sah_system","group-sah_third_party","group-sah_validation","group-sah_wijgmaal","personal-sah4011"],"change":1005814}'
Perforce server info:
2016/12/21 08:39:39 pid 14769 completed .003s 4+0us 0+16io 0+0net 6432k 0pf
"""
md5 = "d0ae06fd40d95180ca403a9c30084a66"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,14769,"2016-12-21 08:39:39","2016-12-21 08:39:39",NULL,0.003,"perforce","~tmp.1482305462.13038.585a2fb6041cc1.60954329","192.168.18.31","SWARM/2016.2/1446446","user-counter","-u swarm-activity-fffec3dd",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2016-12-21 08:39:39')
self.assertEqual(row[4], '2016-12-21 08:39:39')
conn.close()
def testGitFusion(self):
"GitFusion style entry"
# Test parsing of git-fusion style entry
input = r"""
Perforce server info:
2016/10/19 12:01:08 pid 10664 git-fusion-user@GF-TRIGGER-567d67de-962 10.100.104.199 [p4/2016.1/NTX64/1396108] 'user-key git-fusion-reviews-common-lock-owner {"server_id": "p4gf_submit_trigger", "process_id": 5068, "start_time": "2016-10-19 12:01:08"}'
--- lapse .875s
--- rpc msgs/size in+out 2+3/0mb+0mb himarks 523588/523588 snd/rcv .000s/.015s
--- db.nameval
--- pages in+out+cached 6+4+4
--- locks read/write 0/1 rows get+pos+scan put+del 0+0+0 1+0
--- total lock wait+held read/write 0ms+0ms/16ms+15ms
--- db.user
--- pages in+out+cached 4+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- total lock wait+held read/write 0ms+16ms/0ms+0ms
--- db.group
--- pages in+out+cached 7+0+6
--- locks read/write 1/0 rows get+pos+scan put+del 0+3+67 0+0
--- total lock wait+held read/write 0ms+15ms/0ms+0ms
--- db.trigger
--- pages in+out+cached 21+0+20
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+486 0+0
--- total lock wait+held read/write 0ms+47ms/0ms+0ms
--- db.protect
--- pages in+out+cached 282+0+96
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+14495 0+0
--- total lock wait+held read/write 0ms+641ms/0ms+0ms
Perforce server info:
2016/10/19 12:01:09 pid 10664 completed .844s
"""
md5 = "1eec998ae9cc1ce44058f4503a01f2c0"
expected = """INSERT IGNORE INTO tableUse VALUES ("%s",2,"nameval",6,4,4,NULL,NULL,0,1,0,0,0,1,0,0,0,16,15,0,0,16,15,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"protect",282,0,96,NULL,NULL,1,0,0,1,14495,0,0,0,641,0,0,0,641,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"trigger",21,0,20,NULL,NULL,1,0,0,1,486,0,0,0,47,0,0,0,47,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"group",7,0,6,NULL,NULL,1,0,0,3,67,0,0,0,15,0,0,0,15,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"user",4,0,3,NULL,NULL,1,0,1,0,0,0,0,0,16,0,0,0,16,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("%s",2,10664,"2016-10-19 12:01:08","2016-10-19 12:01:09",NULL,0.844,"git-fusion-user","GF-TRIGGER-567d67de-962","10.100.104.199","p4/2016.1/NTX64/1396108","user-key","git-fusion-reviews-common-lock-owner",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2,3,0,0,523588,523588,.000,.015,1,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5, md5, md5, md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(md5, row[0])
self.assertEqual('2016-10-19 12:01:08', row[3])
self.assertEqual('2016-10-19 12:01:09', row[4])
self.assertEqual(0.844, row[6])
conn.close()
def testLongCommand(self):
"Long commands"
input = r"""
Perforce server info:
2015/09/02 16:43:36 pid 4500 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-reconcile -eadf -c 12253 c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt'
Perforce server info:
2015/09/02 16:43:36 pid 4500 completed .187s
Perforce server info:
2015/09/02 16:43:36 pid 4500 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-reconcile -eadf -c 12253 c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt c:\temp\robert-test\test\VEER!-%-#-@-$-&-(-)\fred - Copy.txt'
--- clients/robert-test(W)
--- total lock wait+held read/write 0ms+0ms/0ms+172ms
"""
md5 = "e2bf456007fe305acdae759996dbbeb9"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,4500,"2015-09-02 16:43:36","2015-09-02 16:43:36",NULL,0.187,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-reconcile",
COMMIT;
BEGIN TRANSACTION;
""" % (md5)
self.runTestText(expected, input, longlines=True)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], '2015-09-02 16:43:36')
self.assertEqual(row[4], '2015-09-02 16:43:36')
conn.close()
def testIgnoreErrors(self):
"Long commands"
input = r"""
Perforce server info:
2017/02/09 17:22:35 pid 6582 perforce@irp210-h01 127.0.0.1 [unnamed p4-python script/v81] 'user-client -i'
Perforce server info:
2017/02/09 17:22:35 pid 6582 perforce@perforce.140313515062736 127.0.0.1 [unnamed p4-python script/v81] 'user-sync //perforce.140313515062736/...'
Perforce server error:
Date 2017/02/09 17:22:35:
Pid 6582
Operation: user-sync
Librarian checkout depot/main/01/01/most/sites/have/longer/paths/jam/src/Build.com failed.
open for read: depot/main/01/01/most/sites/have/longer/paths/jam/src/Build.com,v: No such file or directory
"""
expected = """INSERT IGNORE INTO process VALUES ("7b1e9b1001c45ab1abf80312336e5359",2,6582,"2017-02-09 17:22:35",NULL,NULL,NULL,"perforce","irp210-h01","127.0.0.1","unnamed p4-python script/v81","user-client","-i",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
INSERT IGNORE INTO process VALUES ("73d73d8f9a0875f8961d9bec68dcf597",4,6582,"2017-02-09 17:22:35",NULL,NULL,NULL,"perforce","perforce.140313515062736","127.0.0.1","unnamed p4-python script/v81","user-sync","//perforce.140313515062736/...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(2, len(rows))
row = rows[0]
self.assertEqual('2017-02-09 17:22:35', row[3])
self.assertEqual(None, row[4])
conn.close()
def testNetworkEstimates(self):
"Several lines of text"
input = """
Perforce server info:
2017/02/15 10:11:30 pid 4917 bruno@bruno.140451462678608 10.62.185.99 [unnamed p4-python script/v81] 'user-have'
Perforce server info:
2017/02/15 10:11:30 pid 4917 completed .002s 2+0us 0+0io 0+0net 8932k 0pf
Perforce server info:
2017/02/15 10:11:30 pid 4917 bruno@bruno.140451462678608 10.62.185.99 [unnamed p4-python script/v81] 'user-sync //bruno.140451462678608/...'
Perforce server info:
2017/02/15 10:11:30 pid 4917 compute end .020s 16+3us 0+0io 0+0net 8964k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=1/3/0, bytes added/updated=111325/813906
Perforce server info:
2017/02/15 10:11:30 pid 4917 completed .034s 19+4us 0+8io 0+0net 8996k 0pf
"""
expected = """INSERT IGNORE INTO process VALUES ("4964a5f82541f47985f0965ab47c1e39",2,4917,"2017-02-15 10:11:30","2017-02-15 10:11:30",NULL,0.002,"bruno","bruno.140451462678608","10.62.185.99","unnamed p4-python script/v81","user-have",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
INSERT IGNORE INTO process VALUES ("7c65428ac3b32f6f42f84ead5694ffb4",6,4917,"2017-02-15 10:11:30","2017-02-15 10:11:30",0.02,0.034,"bruno","bruno.140451462678608","10.62.185.99","unnamed p4-python script/v81","user-sync","//bruno.140451462678608/...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(2, len(rows))
row = rows[0]
self.assertEqual('2017-02-15 10:11:30', row[3])
self.assertEqual('2017-02-15 10:11:30', row[4])
row = rows[1]
self.assertEqual(0.02, row[5])
self.assertEqual(0.034, row[6])
conn.close()
def testFullTrack(self):
"Full track output with duplicate pids"
input = """
Perforce server info:
2017/02/15 13:48:40 pid 82509 bruno@bruno.140147714537552 10.62.185.101 [unnamed p4-python script/v81] 'user-delete /mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/nasdf'
Perforce server info:
2017/02/15 13:48:40 pid 82509 completed .004s 0+0us 0+16io 0+0net 16640k 0pf
Perforce server info:
2017/02/15 13:48:40 pid 82509 bruno@bruno.140147714537552 10.62.185.101 [unnamed p4-python script/v81] 'user-delete /mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/nasdf'
--- lapse .004s
--- rpc msgs/size in+out 2+4/0mb+0mb himarks 318788/318788 snd/rcv .000s/.004s
--- db.counters
--- pages in+out+cached 2+0+2
--- locks read/write 0/1 rows get+pos+scan put+del 1+0+0 0+0
Perforce server info:
2017/02/15 13:48:40 pid 82509 bruno@bruno.140147714537552 10.62.185.101 [unnamed p4-python script/v81] 'user-edit /mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/TtEVHkGZFv2nCHgbAmp'
Perforce server info:
2017/02/15 13:48:40 pid 82509 completed .004s 0+0us 0+16io 0+0net 16640k 0pf
Perforce server info:
2017/02/15 13:48:40 pid 82509 bruno@bruno.140147714537552 10.62.185.101 [unnamed p4-python script/v81] 'user-edit /mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/TtEVHkGZFv2nCHgbAmp'
--- lapse .004s
--- rpc msgs/size in+out 2+4/0mb+0mb himarks 318788/318788 snd/rcv .000s/.004s
--- db.counters
--- pages in+out+cached 2+0+2
--- locks read/write 0/1 rows get+pos+scan put+del 1+0+0 0+0
--- db.logger
--- pages in+out+cached 2+0+1
--- locks read/write 0/1 rows get+pos+scan put+del 0+0+0 0+0
--- db.user
--- pages in+out+cached 1+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.group
--- pages in+out+cached 1+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 0+2+3 0+0
--- db.domain
--- pages in+out+cached 1+0+4
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.view
--- pages in+out+cached 2+0+4
--- locks read/write 2/0 rows get+pos+scan put+del 0+2+4 0+0
--- db.have
--- pages in+out+cached 1+0+38
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+2 0+0
--- db.rev
--- pages in+out+cached 3+0+29
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+1 0+0
--- db.locks
--- pages in+out+cached 4+3+8
--- locks read/write 1/1 rows get+pos+scan put+del 0+1+1 1+0
--- db.working
--- pages in+out+cached 3+3+4
--- locks read/write 1/1 rows get+pos+scan put+del 1+1+1 1+0
--- db.protect
--- pages in+out+cached 1+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+20 0+0
--- clients/bruno%2E140147714537552%2Eirp210-h04(W)
--- total lock wait+held read/write 0ms+0ms/0ms+4ms
Perforce server info:
2017/02/15 13:48:41 pid 82509 bruno@bruno.140147714537552 10.62.185.101 [unnamed p4-python script/v81] 'user-edit /mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/asdf'
Perforce server info:
2017/02/15 13:48:41 pid 82509 completed .008s 0+0us 0+16io 0+0net 16640k 0pf
"""
expected = """INSERT IGNORE INTO tableUse VALUES ("fd4a77c1ecf5e78926a62481a8a11894",2,"counters",2,0,2,NULL,NULL,0,1,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("fd4a77c1ecf5e78926a62481a8a11894",2,82509,"2017-02-15 13:48:40","2017-02-15 13:48:40",NULL,0.004,"bruno","bruno.140147714537552","10.62.185.101","unnamed p4-python script/v81","user-delete","/mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/nasdf",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2,4,0,0,318788,318788,.000,.004,0,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"clients_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,4,0,0,0,4,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"domain",1,0,4,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"group",1,0,2,NULL,NULL,1,0,0,2,3,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"working",3,3,4,NULL,NULL,1,1,1,1,1,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"rev",3,0,29,NULL,NULL,1,0,0,1,1,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"protect",1,0,2,NULL,NULL,1,0,0,1,20,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"locks",4,3,8,NULL,NULL,1,1,0,1,1,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"user",1,0,2,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"have",1,0,38,NULL,NULL,1,0,0,1,2,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"logger",2,0,1,NULL,NULL,0,1,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"counters",2,0,2,NULL,NULL,0,1,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,"view",2,0,4,NULL,NULL,2,0,0,2,4,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("f41162fe6ebc4488ed2f6096d2ed59ff.14",14,82509,"2017-02-15 13:48:40","2017-02-15 13:48:40",NULL,0.004,"bruno","bruno.140147714537552","10.62.185.101","unnamed p4-python script/v81","user-edit","/mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/TtEVHkGZFv2nCHgbAmp",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2,4,0,0,318788,318788,.000,.004,0,NULL);
INSERT IGNORE INTO process VALUES ("f859808fd6ce0a27e20d6f0061a8bae5",58,82509,"2017-02-15 13:48:41","2017-02-15 13:48:41",NULL,0.008,"bruno","bruno.140147714537552","10.62.185.101","unnamed p4-python script/v81","user-edit","/mnt/ebc-201/bench_tests/bruno.140147714537552/Jam/REL2.1/src/asdf",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
conn.close()
def testRmtFileFetches(self):
"These get duplicate pids in same second and have no completed record"
input = """
Perforce server info:
2017/03/06 11:53:50 pid 113249 serviceUser@unknown 10.62.185.99 [p4d/2016.2/LINUX26X86_64/1468155] 'rmt-FileFetch'
--- rpc msgs/size in+out 0+2/0mb+0mb himarks 318788/318788 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 2+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
Perforce server info:
2017/03/06 11:53:50 pid 113249 serviceUser@unknown 10.62.185.99 [p4d/2016.2/LINUX26X86_64/1468155] 'rmt-FileFetch'
--- rpc msgs/size in+out 0+2/0mb+0mb himarks 318788/318788 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
Perforce server info:
2017/03/06 11:53:50 pid 113249 serviceUser@unknown 10.62.185.99 [p4d/2016.2/LINUX26X86_64/1468155] 'rmt-FileFetch'
--- rpc msgs/size in+out 0+2/0mb+0mb himarks 318788/318788 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+2
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
"""
expected = """INSERT IGNORE INTO tableUse VALUES ("bea947227d9ec7f4300a0ea889886934",3,"user",2,0,2,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("bea947227d9ec7f4300a0ea889886934",3,113249,"2017-03-06 11:53:50","2017-03-06 11:53:50",NULL,NULL,"serviceUser","unknown","10.62.185.99","p4d/2016.2/LINUX26X86_64/1468155","rmt-FileFetch",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,2,0,0,318788,318788,.000,.000,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("bea947227d9ec7f4300a0ea889886934.10",10,"user",1,0,2,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("bea947227d9ec7f4300a0ea889886934.10",10,113249,"2017-03-06 11:53:50","2017-03-06 11:53:50",NULL,NULL,"serviceUser","unknown","10.62.185.99","p4d/2016.2/LINUX26X86_64/1468155","rmt-FileFetch",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,2,0,0,318788,318788,.000,.000,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("bea947227d9ec7f4300a0ea889886934.17",17,"user",1,0,2,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("bea947227d9ec7f4300a0ea889886934.17",17,113249,"2017-03-06 11:53:50","2017-03-06 11:53:50",NULL,NULL,"serviceUser","unknown","10.62.185.99","p4d/2016.2/LINUX26X86_64/1468155","rmt-FileFetch",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,2,0,0,318788,318788,.000,.000,1,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
conn.close()
def testRmtJournals(self):
"These also get duplicate pids in same second and have no completed record"
input = """
Perforce server info:
2018/04/12 10:00:01 pid 68364 serviceUser@unknown 10.10.18.53 [p4d/2016.2/LINUX26X86_64/1487173] 'rmt-Journal'
--- lapse .051s
--- usage 1+0us 0+0io 0+0net 3924k 0pf
--- rpc msgs/size in+out 0+2/0mb+0mb himarks 318788/318788 snd/rcv .000s/.000s
--- db.counters
--- pages in+out+cached 3+0+2
--- locks read/write 3/0 rows get+pos+scan put+del 3+0+0 0+0
Perforce server info:
2018/04/12 10:00:01 pid 68364 serviceUser@unknown 10.10.18.53 [p4d/2016.2/LINUX26X86_64/1487173] 'rmt-Journal'
--- lapse .101s
--- usage 1+0us 0+0io 0+0net 3924k 0pf
--- rpc msgs/size in+out 0+2/0mb+0mb himarks 318788/318788 snd/rcv .000s/.000s
--- db.counters
--- pages in+out+cached 4+0+2
--- locks read/write 4/0 rows get+pos+scan put+del 4+0+0 0+0
"""
expected = """INSERT IGNORE INTO process VALUES ("245384318ebd4fb4bdbf5e023f52cdb0",3,68364,"2018-04-12 10:00:01","2018-04-12 10:00:01",NULL,0.051,"serviceUser","unknown","10.10.18.53","p4d/2016.2/LINUX26X86_64/1487173","rmt-Journal",NULL,1,0,0,0,0,0,3924,0,0,2,0,0,318788,318788,.000,.000,1,NULL);
INSERT IGNORE INTO process VALUES ("245384318ebd4fb4bdbf5e023f52cdb0.12",12,68364,"2018-04-12 10:00:01","2018-04-12 10:00:01",NULL,0.101,"serviceUser","unknown","10.10.18.53","p4d/2016.2/LINUX26X86_64/1487173","rmt-Journal",NULL,1,0,0,0,0,0,3924,0,0,2,0,0,318788,318788,.000,.000,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("245384318ebd4fb4bdbf5e023f52cdb0",3,"counters",3,0,2,NULL,NULL,3,0,3,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("245384318ebd4fb4bdbf5e023f52cdb0.12",12,"counters",4,0,2,NULL,NULL,4,0,4,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(2, len(rows))
conn.close()
def testChanges(self):
"These get duplicate pids for change -o"
input = """
Perforce server info:
2016/10/19 14:53:48 pid 4496 lcheng@lcheng 10.100.72.195 [P4V/NTX64/2014.1/888424/v76] 'user-change -o'
Perforce server info:
2016/10/19 14:53:48 pid 4496 completed .015s
Perforce server info:
2016/10/19 14:53:48 pid 4496 lcheng@lcheng 10.100.72.195 [P4V/NTX64/2014.1/888424/v76] 'user-change -o'
--- lapse .015s
--- rpc msgs/size in+out 0+1/0mb+0mb himarks 523588/64836 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.group
--- pages in+out+cached 1+0+7
--- locks read/write 1/0 rows get+pos+scan put+del 0+6+11 0+0
Perforce server info:
2016/10/19 14:53:48 pid 4496 lcheng@lcheng 10.100.72.195 [P4V/NTX64/2014.1/888424/v76] 'user-change -o'
Perforce server info:
2016/10/19 14:53:48 pid 4496 completed .016s
Perforce server info:
2016/10/19 14:53:48 pid 4496 lcheng@lcheng 10.100.72.195 [P4V/NTX64/2014.1/888424/v76] 'user-change -o'
--- lapse .016s
--- rpc msgs/size in+out 0+1/0mb+0mb himarks 523588/64836 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.group
--- pages in+out+cached 1+0+7
--- locks read/write 1/0 rows get+pos+scan put+del 0+6+11 0+0
"""
expected = """INSERT IGNORE INTO tableUse VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c",2,"group",1,0,7,NULL,NULL,1,0,0,6,11,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c",2,"user",1,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c.18",18,"group",1,0,7,NULL,NULL,1,0,0,6,11,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c.18",18,"user",1,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c",2,4496,"2016-10-19 14:53:48","2016-10-19 14:53:48",NULL,0.015,"lcheng","lcheng","10.100.72.195","P4V/NTX64/2014.1/888424/v76","user-change","-o",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,0,0,523588,64836,.000,.000,0,NULL);
INSERT IGNORE INTO process VALUES ("9b2bf87ce1b8e88d0d89cf44cffc4a8c.18",18,4496,"2016-10-19 14:53:48","2016-10-19 14:53:48",NULL,0.016,"lcheng","lcheng","10.100.72.195","P4V/NTX64/2014.1/888424/v76","user-change","-o",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,0,0,523588,64836,.000,.000,0,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
# Test with SQL
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(2, len(rows))
conn.close()
def testDuplicateCmds(self):
"These pids re-used on same connection for same command within same second - so appear to be duplicates"
input = """
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
Perforce server info:
2017/12/07 15:00:27 pid 148889 completed .001s 1+0us 0+0io 0+0net 3924k 0pf
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
--- lapse .002s
--- usage 2+0us 0+8io 0+0net 3924k 0pf
--- rpc msgs/size in+out 0+1/0mb+0mb himarks 318788/523588 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
Perforce server info:
2017/12/07 15:00:27 pid 148889 completed .001s 1+0us 0+0io 0+0net 3924k 0pf
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
--- lapse .002s
--- usage 2+0us 0+8io 0+0net 3924k 0pf
--- rpc msgs/size in+out 0+1/0mb+0mb himarks 318788/523588 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
Perforce server info:
2017/12/07 15:00:27 pid 148889 completed .001s 1+0us 0+0io 0+0net 3924k 0pf
Perforce server info:
2017/12/07 15:00:27 pid 148889 fred.smith@FRED-WS 10.10.16.172/10.10.65.237 [P4VS/2017.2.159.0319] 'user-fstat -Olhp x:/dev/Client/IService.cs'
--- lapse .002s
--- usage 2+0us 0+8io 0+0net 3924k 0pf
--- rpc msgs/size in+out 0+1/0mb+0mb himarks 318788/523588 snd/rcv .000s/.000s
--- db.user
--- pages in+out+cached 1+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
"""
expected = """INSERT IGNORE INTO tableUse VALUES ("5bfccae999a84d9dd29185a04da18692",2,"user",1,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("5bfccae999a84d9dd29185a04da18692.16",16,"user",1,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("5bfccae999a84d9dd29185a04da18692.29",29,"user",1,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO process VALUES ("5bfccae999a84d9dd29185a04da18692",2,148889,"2017-12-07 15:00:27","2017-12-07 15:00:27",NULL,0.001,"fred.smith","FRED-WS","10.10.16.172/10.10.65.237","P4VS/2017.2.159.0319","user-fstat","-Olhp x:/dev/Client/IService.cs",2,0,0,8,0,0,3924,0,0,1,0,0,318788,523588,.000,.000,0,NULL);
INSERT IGNORE INTO process VALUES ("5bfccae999a84d9dd29185a04da18692.16",16,148889,"2017-12-07 15:00:27","2017-12-07 15:00:27",NULL,0.001,"fred.smith","FRED-WS","10.10.16.172/10.10.65.237","P4VS/2017.2.159.0319","user-fstat","-Olhp x:/dev/Client/IService.cs",2,0,0,8,0,0,3924,0,0,1,0,0,318788,523588,.000,.000,0,NULL);
INSERT IGNORE INTO process VALUES ("5bfccae999a84d9dd29185a04da18692.29",29,148889,"2017-12-07 15:00:27","2017-12-07 15:00:27",NULL,0.001,"fred.smith","FRED-WS","10.10.16.172/10.10.65.237","P4VS/2017.2.159.0319","user-fstat","-Olhp x:/dev/Client/IService.cs",2,0,0,8,0,0,3924,0,0,1,0,0,318788,523588,.000,.000,0,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
c.execute('SELECT * from tableUse;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
conn.close()
def testNonUtf8(self):
"unicode errors - particularly in python3"
# Same as above with invalid Unicode strings
input = """
Perforce server info:
2017/12/07 15:26:59 pid 86264 john.smith@TEST_WS 10.10.16.172/10.10.48.217 [P4V/NTX64/2017.3/1592764/v83] 'user-fstat -Olhp x:\ws\data\interior.xml x:\ws\data\y2017\� x:\ws\data\y2017\content_audio.xml'
Perforce server info:
2017/12/07 15:26:59 pid 86264 completed .004s 3+1us 0+0io 0+0net 6996k 0pf
"""
md5 = "28e033f1140f564baa7e2a8eda812805"
expected = r"""INSERT IGNORE INTO process VALUES ("%s",2,86264,"2017-12-07 15:26:59","2017-12-07 15:26:59",NULL,0.004,"john.smith","TEST_WS","10.10.16.172/10.10.48.217","P4V/NTX64/2017.3/1592764/v83","user-fstat","-Olhp x:\\ws\\data\\interior.xml x:\\ws\\data\\y2017\\� x:\\ws\\data\\y2017\\content_audio.xml",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
if python3:
md5 = "3ecb25a0dee3b912913f5106966ef9f1"
expected = r"""INSERT IGNORE INTO process VALUES ("%s",2,86264,"2017-12-07 15:26:59","2017-12-07 15:26:59",NULL,0.004,"john.smith","TEST_WS","10.10.16.172/10.10.48.217","P4V/NTX64/2017.3/1592764/v83","user-fstat","-Olhp x:\\ws\\data\\interior.xml x:\\ws\\data\\y2017\\� x:\\ws\\data\\y2017\\content_audio.xml",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % md5
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertEqual(row[0], md5)
self.assertEqual(row[3], u'2017-12-07 15:26:59')
conn.close()
def testPeeks(self):
"Process peek count records"
# Same as above with invalid Unicode strings
input = r"""
Perforce server info:
2017/12/07 15:00:01 pid 145999 john.smith@john.smith_LON 10.10.16.171/10.10.48.88 [P4VS/2017.2.159.0319] 'user-fstat -Olhp -m 100000 x:/ws/*'
Perforce server info:
2017/12/07 15:00:01 pid 145999 completed .007s 0+3us 0+8io 0+0net 3920k 0pf
Perforce server info:
2017/12/07 15:00:01 pid 145999 john.smith@john.smith_LON 10.10.16.171/10.10.48.88 [P4VS/2017.2.159.0319] 'user-fstat -Olhp -m 100000 x:/ws/*'
--- lapse .007s
--- usage 1+3us 0+24io 0+0net 3920k 0pf
--- rpc msgs/size in+out 2+3/0mb+0mb himarks 318788/2096452 snd/rcv .000s/.002s
--- db.user
--- pages in+out+cached 4+0+3
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.group
--- pages in+out+cached 16+0+15
--- locks read/write 1/0 rows get+pos+scan put+del 0+18+35 0+0
--- db.domain
--- pages in+out+cached 5+0+4
--- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
--- db.view
--- pages in+out+cached 3+0+6
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+13 0+0
--- db.have
--- pages in+out+cached 8+0+7
--- locks read/write 0/0 rows get+pos+scan put+del 0+1+1 0+0
--- peek count 1 wait+held total/max 2ms+3ms/4ms+5ms
--- db.revdx
--- pages in+out+cached 7+0+6
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+1 0+0
--- db.revhx
--- pages in+out+cached 8+0+7
--- locks read/write 1/0 rows get+pos+scan put+del 0+1+1 0+0
--- db.rev
--- pages in+out+cached 1+0+1
--- locks read/write 1/0 rows get+pos+scan put+del 0+0+0 0+0
--- db.revtx
--- pages in+out+cached 1+0+1
--- locks read/write 1/0 rows get+pos+scan put+del 0+0+0 0+0
--- db.locks
--- pages in+out+cached 1+0+1
--- locks read/write 0/0 rows get+pos+scan put+del 0+0+0 0+0
--- peek count 1 wait+held total/max 2ms+3ms/4ms+5ms
"""
md5 = "38faa33c7e0e6f6494e6d009b15cfc65"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,145999,"2017-12-07 15:00:01","2017-12-07 15:00:01",NULL,0.007,"john.smith","john.smith_LON","10.10.16.171/10.10.48.88","P4VS/2017.2.159.0319","user-fstat","-Olhp -m 100000 x:/ws/*",1,3,0,24,0,0,3920,0,2,3,0,0,318788,2096452,.000,.002,0,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"domain",5,0,4,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"group",16,0,15,NULL,NULL,1,0,0,18,35,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"have",8,0,7,NULL,NULL,0,0,0,1,1,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,2,3,4,5,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"locks",1,0,1,NULL,NULL,0,0,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,2,3,4,5,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"rev",1,0,1,NULL,NULL,1,0,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"revdx",7,0,6,NULL,NULL,1,0,0,1,1,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"revhx",8,0,7,NULL,NULL,1,0,0,1,1,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"revtx",1,0,1,NULL,NULL,1,0,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"user",4,0,3,NULL,NULL,1,0,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"view",3,0,6,NULL,NULL,1,0,0,1,13,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5, md5, md5, md5, md5, md5, md5, md5, md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
if python3:
self.assertEqual(row[0], md5)
else:
self.assertEqual(row[0], md5)
self.assertEqual(row[3], u'2017-12-07 15:00:01')
conn.close()
def testSyncMultiArgMetaDb(self):
"""Process multiple meta/db entries and intermediate
This occurs when you run a sync with multiple arguments. Intermediate records are output for
every argument and include "compute end" and "meta/db" records. They need to be treated
as updates to the single sync command. Could probably even be ignored as the final one will
contain the final results.
In any case, it is important that they are not treated as multiple individual sync commands!
"""
input = r"""
Perforce server info:
2017/12/07 15:00:01 pid 145941 builder@LON 10.10.16.171/10.10.20.195 [AutoWorker/1.0.0.0] 'user-sync //assets/level/instances.xml'
Perforce server info:
2017/12/07 15:00:01 pid 145941 builder@LON 10.10.16.171/10.10.20.195 [AutoWorker/1.0.0.0] 'user-sync //assets/level/instances.xml'
--- meta/db(R)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Perforce server info:
2017/12/07 15:00:01 pid 145941 compute end .007s 3+1us 16+112io 0+0net 4452k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=0/0/0, bytes added/updated=0/0
Perforce server info:
2017/12/07 15:00:01 pid 145941 builder@LON 10.10.16.171/10.10.20.195 [AutoWorker/1.0.0.0] 'user-sync //assets/level/instances.xml'
--- meta/db(R)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Perforce server info:
2017/12/07 15:00:01 pid 145941 compute end .007s 3+1us 16+128io 0+0net 4452k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=0/0/0, bytes added/updated=0/0
Perforce server info:
2017/12/07 15:00:01 pid 145941 builder@LON 10.10.16.171/10.10.20.195 [AutoWorker/1.0.0.0] 'user-sync //assets/level/instances.xml'
--- meta/db(R)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Perforce server info:
2017/12/07 15:00:01 pid 145941 compute end .008s 4+1us 16+144io 0+0net 4452k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=0/0/0, bytes added/updated=0/0
Perforce server info:
2017/12/07 15:00:01 pid 145941 builder@LON 10.10.16.171/10.10.20.195 [AutoWorker/1.0.0.0] 'user-sync //assets/level/instances.xml'
--- meta/db(R)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Perforce server info:
2017/12/07 15:00:01 pid 145941 compute end .008s 4+1us 16+160io 0+0net 4452k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=0/0/0, bytes added/updated=0/0
Perforce server info:
--- meta/db(R)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Perforce server info:
2017/12/07 15:00:01 pid 145941 compute end .110s 77+25us 112+3120io 0+0net 4964k 0pf
Perforce server info:
Server network estimates: files added/updated/deleted=0/0/0, bytes added/updated=0/0
Perforce server info:
2017/12/07 15:00:01 pid 145941 completed .111s 77+25us 112+3136io 0+0net 4964k 0pf
"""
md5 = "026c2d4135085764d23fd21f41d30f77"
expected = """INSERT IGNORE INTO process VALUES ("%s",5,145941,"2017-12-07 15:00:01","2017-12-07 15:00:01",0.14,0.111,"builder","LON","10.10.16.171/10.10.20.195","AutoWorker/1.0.0.0","user-sync","//assets/level/instances.xml",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",5,"meta_db_R",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
if python3:
self.assertEqual(row[0], md5)
else:
self.assertEqual(row[0], md5)
self.assertEqual(row[3], u'2017-12-07 15:00:01')
conn.close()
def testTriggerEntries(self):
"""Process Trigger lapse times which are logged at the end of lines with an (inconsistently formatted)
lapse entry on the next line, but only if configurable track=1 is set.
"""
input = r"""
Perforce server info:
2017/12/07 15:00:21 pid 148469 Fred@LONWS 10.40.16.14/10.40.48.29 [3DSMax/1.0.0.0] 'user-change -i' trigger swarm.changesave
lapse .044s
Perforce server info:
2017/12/07 15:00:21 pid 148469 completed .413s 7+4us 0+584io 0+0net 4580k 0pf
Perforce server info:
2017/12/07 15:00:21 pid 148469 Fred@LONWS 10.40.16.14/10.40.48.29 [3DSMax/1.0.0.0] 'user-change -i'
--- lapse .413s
--- usage 7+4us 0+592io 0+0net 4580k 0pf
--- rpc msgs/size in+out 3+5/0mb+0mb himarks 318788/2096452 snd/rcv .000s/.052s
--- db.counters
--- pages in+out+cached 6+3+2
--- locks read/write 0/2 rows get+pos+scan put+del 2+0+0 1+0
"""
md5 = "25aeba7a5658170fea61117076fa00d5"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,148469,"2017-12-07 15:00:21","2017-12-07 15:00:21",NULL,0.413,"Fred","LONWS","10.40.16.14/10.40.48.29","3DSMax/1.0.0.0","user-change","-i",7,4,0,592,0,0,4580,0,3,5,0,0,318788,2096452,.000,.052,0,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"counters",6,3,2,NULL,NULL,0,2,2,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"trigger_swarm.changesave",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0.044);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
if python3:
self.assertEqual(row[0], md5)
else:
self.assertEqual(row[0], md5)
self.assertEqual(row[3], u'2017-12-07 15:00:21')
conn.close()
def testAPIStrangeNames(self):
"""Where API/version id also includes [] in the string
"""
input = r"""
Perforce server info:
2017/12/07 15:00:21 pid 148469 Fred@LONWS 10.40.16.14 [p4buildfarm [PY3.6.4/P4PY2017.2.MAIN/API2017.2/1611521]/v83] 'user-change -i'
Perforce server info:
2017/12/07 15:00:21 pid 148469 completed .413s 7+4us 0+584io 0+0net 4580k 0pf
Perforce server info:
2017/12/07 15:00:21 pid 148469 Fred@LONWS 10.40.16.14 [p4buildfarm [PY3.6.4/P4PY2017.2.MAIN/API2017.2/1611521]/v83] 'user-change -i'
--- lapse .413s
--- usage 7+4us 0+592io 0+0net 4580k 0pf
--- rpc msgs/size in+out 3+5/0mb+0mb himarks 318788/2096452 snd/rcv .000s/.052s
--- db.counters
--- pages in+out+cached 6+3+2
--- locks read/write 0/2 rows get+pos+scan put+del 2+0+0 1+0
"""
md5 = "7fb3212fe8d224a4824c1fadc8b1cbff"
expected = """INSERT IGNORE INTO process VALUES ("%s",2,148469,"2017-12-07 15:00:21","2017-12-07 15:00:21",NULL,0.413,"Fred","LONWS","10.40.16.14","p4buildfarm [PY3.6.4/P4PY2017.2.MAIN/API2017.2/1611521]/v83","user-change","-i",7,4,0,592,0,0,4580,0,3,5,0,0,318788,2096452,.000,.052,0,NULL);
INSERT IGNORE INTO tableUse VALUES ("%s",2,"counters",6,3,2,NULL,NULL,0,2,2,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
""" % (md5, md5)
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
if python3:
self.assertEqual(row[0], md5)
else:
self.assertEqual(row[0], md5)
self.assertEqual(row[3], u'2017-12-07 15:00:21')
conn.close()
def testEdgeLog(self):
"""Process edge server log
"""
input = r"""
Perforce server info:
2018/06/01 04:29:43 pid 55997 svc0@unknown background [p4d/2018.1/DARWIN90X86_64/1660568] 'pull -I 100 -b 1'
--- db.counters
--- pages in+out+cached 2+0+2
--- locks read/write 0/1 rows get+pos+scan put+del 1+0+0 0+0
--- replica/pull(W)
--- total lock wait+held read/write 0ms+0ms/0ms+1ms
Perforce server info:
2018/06/01 04:29:43 pid 55997 svc0@unknown background [p4d/2018.1/DARWIN90X86_64/1660568] 'pull -I 100 -b 1'
--- db.counters
--- pages in+out+cached 4+3+2
--- locks read/write 0/2 rows get+pos+scan put+del 0+0+0 1+1
--- replica/pull(W)
--- total lock wait+held read/write 0ms+0ms/0ms+1ms
Perforce server info:
2018/06/01 04:29:43 pid 55997 svc0@unknown background [p4d/2018.1/DARWIN90X86_64/1660568] 'pull -I 100 -b 1'
--- lapse .001s
--- db.counters
--- pages in+out+cached 2+3+2
--- locks read/write 0/1 rows get+pos+scan put+del 1+0+0 1+0
--- db.change
--- pages in+out+cached 4+3+2
--- locks read/write 0/1 rows get+pos+scan put+del 0+0+0 1+0
--- db.changex
--- pages in+out+cached 4+3+2
--- locks read/write 0/1 rows get+pos+scan put+del 0+0+0 1+0
--- db.desc
--- pages in+out+cached 4+3+2
--- locks read/write 0/1 rows get+pos+scan put+del 0+0+0 1+0
--- replica/pull(W)
--- total lock wait+held read/write 0ms+0ms/0ms+0ms
Pull command 55998 xfering //depot/some_file#1.1 (add, text)
2018/06/01 04:29:44 544730000 pid 55998: connected to central server
Perforce server info:
2018/06/01 04:29:44 pid 55998 svc0@unknown background [p4d/2018.1/DARWIN90X86_64/1660568] 'pull -u -i 1 -b 1'
--- rdb.lbr
--- pages in+out+cached 7+4+2
--- locks read/write 0/3 rows get+pos+scan put+del 1+1+4 1+1
"""
expected = """INSERT IGNORE INTO process VALUES ("44c92f3be809fd15dfc26cc8fb359216",38,55998,"2018-06-01 04:29:44","2018-06-01 04:29:44",NULL,NULL,"svc0","unknown","background","p4d/2018.1/DARWIN90X86_64/1660568","pull","-u -i 1 -b 1",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,2,NULL);
INSERT IGNORE INTO process VALUES ("9e39beedee815db46bb4c870c11a0b8d",2,55997,"2018-06-01 04:29:43","2018-06-01 04:29:43",NULL,NULL,"svc0","unknown","background","p4d/2018.1/DARWIN90X86_64/1660568","pull","-I 100 -b 1",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO process VALUES ("9e39beedee815db46bb4c870c11a0b8d.10",10,55997,"2018-06-01 04:29:43","2018-06-01 04:29:43",NULL,NULL,"svc0","unknown","background","p4d/2018.1/DARWIN90X86_64/1660568","pull","-I 100 -b 1",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO process VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,55997,"2018-06-01 04:29:43","2018-06-01 04:29:43",NULL,0.001,"svc0","unknown","background","p4d/2018.1/DARWIN90X86_64/1660568","pull","-I 100 -b 1",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d",2,"counters",2,0,2,NULL,NULL,0,1,1,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d",2,"replica_pull_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,1,0,0,0,1,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.10",10,"counters",4,3,2,NULL,NULL,0,2,0,0,0,1,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.10",10,"replica_pull_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,1,0,0,0,1,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,"change",4,3,2,NULL,NULL,0,1,0,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,"changex",4,3,2,NULL,NULL,0,1,0,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,"counters",2,3,2,NULL,NULL,0,1,1,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,"desc",4,3,2,NULL,NULL,0,1,0,0,0,1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("9e39beedee815db46bb4c870c11a0b8d.18",18,"replica_pull_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,0,0,0,0,0,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("44c92f3be809fd15dfc26cc8fb359216",38,"rdb.lbr",7,4,2,NULL,NULL,0,3,1,1,4,1,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process;')
rows = c.fetchall()
self.assertEqual(4, len(rows))
row = rows[0]
self.assertEqual(row[3], u'2018-06-01 04:29:43')
conn.close()
def testSubmitEntries(self):
"""Submits are multi-phase and we need to match these up.
"""
input = r"""
Perforce server info:
2018/06/10 23:30:06 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'user-submit -i'
Perforce server info:
2018/06/10 23:30:07 pid 25568 completed .178s 96+17us 0+208io 0+0net 15668k 0pf
Perforce server info:
2018/06/10 23:30:07 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'dm-SubmitChange'
Perforce server info:
2018/06/10 23:30:07 pid 25568 compute end .252s 35+6us 0+8io 0+0net 49596k 0pf
Perforce server info:
2018/06/10 23:30:08 pid 25568 completed 1.38s 490+165us 0+178824io 0+0net 127728k 0pf
Perforce server info:
2018/06/10 23:30:08 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'dm-CommitSubmit'
Perforce server info:
2018/06/10 23:30:08 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'dm-CommitSubmit'
--- meta/commit(W)
--- total lock wait+held read/write 0ms+0ms/0ms+795ms
Perforce server info:
2018/06/10 23:30:08 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'dm-CommitSubmit'
--- clients/MCM_client_184%2E51%2E33%2E29_prod_prefix1(W)
--- total lock wait+held read/write 0ms+0ms/0ms+1367ms
Perforce server info:
2018/06/10 23:30:09 pid 25568 completed 1.38s 34+61us 59680+59904io 0+0net 127728k 1pf
Perforce server info:
2018/06/10 23:30:08 pid 25568 fred@lon_ws 10.1.2.3 [p4/2016.2/LINUX26X86_64/1598668] 'dm-CommitSubmit'
--- db.integed
--- total lock wait+held read/write 0ms+0ms/0ms+795ms
--- db.archmap
--- total lock wait+held read/write 0ms+0ms/0ms+780ms
"""
expected = """INSERT IGNORE INTO process VALUES ("128e10d7fe570c2d2f5f7f03e1186827",15,25568,"2018-06-10 23:30:08","2018-06-10 23:30:09",NULL,1.38,"fred","lon_ws","10.1.2.3","p4/2016.2/LINUX26X86_64/1598668","dm-CommitSubmit",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO process VALUES ("465f0a630b021d3c695e90924a757b75",2,25568,"2018-06-10 23:30:06","2018-06-10 23:30:07",NULL,0.178,"fred","lon_ws","10.1.2.3","p4/2016.2/LINUX26X86_64/1598668","user-submit","-i",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
INSERT IGNORE INTO process VALUES ("78dbd54644e624a9c6f5c338a0864d2a",7,25568,"2018-06-10 23:30:07","2018-06-10 23:30:08",0.252,1.38,"fred","lon_ws","10.1.2.3","p4/2016.2/LINUX26X86_64/1598668","dm-SubmitChange",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
INSERT IGNORE INTO tableUse VALUES ("128e10d7fe570c2d2f5f7f03e1186827",15,"archmap",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,780,0,0,0,780,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("128e10d7fe570c2d2f5f7f03e1186827",15,"clients_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,1367,0,0,0,1367,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("128e10d7fe570c2d2f5f7f03e1186827",15,"integed",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,795,0,0,0,795,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("128e10d7fe570c2d2f5f7f03e1186827",15,"meta_commit_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,795,0,0,0,795,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process order by processkey;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
row = rows[0]
self.assertEqual("128e10d7fe570c2d2f5f7f03e1186827", row[0])
self.assertEqual(u'2018-06-10 23:30:08', row[3])
conn.close()
def testBlockWithLocksAcquired(self):
"""Some blocks have this line before track info output
"""
input = r"""
Perforce server info:
2018/09/06 06:00:02 pid 22245 auto@archive_auto 127.0.0.1 [archive/v60] 'user-revert /usr/local/arch/datastore/...'
Perforce server info:
2018/09/06 06:00:02 pid 22245 completed 6.92s 6901+4us 32+8io 0+0net 19996k 0pf
Perforce server info:
2018/09/06 06:00:02 pid 22245 auto@archive_auto 127.0.0.1 [archive/v60] 'user-revert /usr/local/arch/datastore/...'
locks acquired by blocking after 3 non-blocking attempts
--- db.resolve
--- total lock wait+held read/write 23792ms+3ms/2ms+1ms
--- max lock wait+held read/write 23792ms+3ms/2ms+1ms
--- db.protect
--- total lock wait+held read/write 4ms+6875ms/5ms+6ms
--- clients/archive_ghostdir%2Etapioca%2Edata_ump_72%2E246%2E96%2E199(W)
--- total lock wait+held read/write 0ms+0ms/0ms+23800ms
"""
expected = """INSERT IGNORE INTO process VALUES ("f7d483631e94d16adde6c5306be15fbe",2,22245,"2018-09-06 06:00:02","2018-09-06 06:00:02",NULL,6.92,"auto","archive_auto","127.0.0.1","archive/v60","user-revert","/usr/local/arch/datastore/...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL);
INSERT IGNORE INTO tableUse VALUES ("f7d483631e94d16adde6c5306be15fbe",2,"resolve",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,23792,3,2,1,23792,3,2,1,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f7d483631e94d16adde6c5306be15fbe",2,"protect",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,4,6875,5,6,4,6875,5,6,NULL,NULL,NULL,NULL,NULL,NULL);
INSERT IGNORE INTO tableUse VALUES ("f7d483631e94d16adde6c5306be15fbe",2,"clients_W",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,0,23800,0,0,0,23800,NULL,NULL,NULL,NULL,NULL,NULL);
COMMIT;
BEGIN TRANSACTION;
"""
self.runTestText(expected, input)
conn = self.runTestSQL(input)
c = conn.cursor()
c.execute('SELECT * from Process order by processkey;')
rows = c.fetchall()
self.assertEqual(1, len(rows))
row = rows[0]
self.assertEqual("f7d483631e94d16adde6c5306be15fbe", row[0])
self.assertEqual(u'2018-09-06 06:00:02', row[3])
c.execute('SELECT processkey, lineNumber, tableName, pagesIn, pagesOut, pagesCached, readLocks, writeLocks, '
'getRows, posRows, scanRows, putRows, delRows, totalReadWait, totalReadHeld, totalWriteWait, '
'totalWriteHeld, maxReadWait, maxReadHeld, maxWriteWait, maxWriteHeld, '
'peekCount, totalPeekWait, totalPeekHeld, maxPeekWait, maxPeekHeld, triggerLapse from tableUse '
'ORDER by tableName;')
rows = c.fetchall()
self.assertEqual(3, len(rows))
row = rows[1]
self.assertEqual(row[2], 'protect') # tableName
self.assertEqual(row[13], 4) # totalReadWait
self.assertEqual(row[14], 6875) # totalReadHeld
self.assertEqual(row[15], 5) # totalWriteWait
self.assertEqual(row[16], 6) # totalWriteHeld
row = rows[2]
self.assertEqual(row[0], 'f7d483631e94d16adde6c5306be15fbe')
self.assertEqual(row[1], 2) # lineNumber
self.assertEqual(row[2], 'resolve') # tableName
self.assertEqual(row[13], 23792) # totalReadWait
self.assertEqual(row[14], 3) # totalReadHeld
self.assertEqual(row[15], 2) # totalWriteWait
self.assertEqual(row[16], 1) # totalWriteHeld
self.assertEqual(row[17], 23792) # maxReadWait
self.assertEqual(row[18], 3) # maxReadHeld
self.assertEqual(row[19], 2) # maxWriteWait
self.assertEqual(row[20], 1) # maxWriteHeld
conn.close()
# Remote depots!!
# rcowham@rcowham:~/Perforce/rcowham.p4d$ p4 -Ztrack=1 sizes -sh //...
# //... 11137049 files 2.7T bytes
# --- lapse 33.9s
# --- usage 15628+2800us 5314928+0io 0+0net 108884k 0pf
# --- rpc msgs/size in+out 2+4/0mb+0mb himarks 795800/523796 snd/rcv .000s/.191s
# --- rmt bld-ondemand.perforce.com:9991
# --- lapse 2.90s
# --- probes+rows 1+86803
# --- rmt workshop.perforce.com:1666
# --- lapse 1.61s
# --- probes+rows 1+189411
# --- rmt server.perforce.com:1951
# --- lapse .446s
# --- probes+rows 1+50278
# --- db.user
# --- pages in+out+cached 4+0+3
# --- locks read/write 1/0 rows get+pos+scan put+del 1+0+0 0+0
if __name__ == '__main__':
unittest.main()
| # | Change | User | Description | Committed | |
|---|---|---|---|---|---|
| #3 | 26269 | Robert Cowham |
Added new table fields parsed: pagesSplitInternal, pagesSplitLeaf Indicates possible high CPU due to tabl expansion |
||
| #2 | 26077 | Robert Cowham | Fix test failing in python 2 | ||
| #1 | 25216 | Robert Cowham | Branch files to Workshop mandated path for project | ||
| //guest/perforce_software/utils/log_analyzer/psla/test_log2sql.py | |||||
| #20 | 24991 | Robert Cowham |
Add tests for all TableUse values. Fix 'locks acquired by blocking after' processing |
||
| #19 | 24674 | Robert Cowham | handle [] in api names | ||
| #18 | 24641 | Robert Cowham | Explicit sql column testing | ||
| #17 | 24639 | sgcommon |
Correct usage and rpc tracking values. Per the schema for the process table, the values were being written in the wrong order. They are now written in the correct order making queries return the expected results for usage and rpc track values. Test data updated accordingly. |
||
| #16 | 24323 | Robert Cowham | Test for dm-CommitSubmit entries | ||
| #15 | 24288 | Robert Cowham | Note some new log entries for remote depots for tests etc | ||
| #14 | 24256 | Robert Cowham | Make sure columns are correct | ||
| #13 | 24171 | Robert Cowham | Remove what may be invalid data so we can ignore it. | ||
| #12 | 24169 | Robert Cowham | More replica pull tweaks | ||
| #11 | 24168 | Robert Cowham |
Process edge server log pull records Some tweaks to tests required. |
||
| #10 | 23934 | Robert Cowham | Clean database and auto-create on log upload | ||
| #9 | 23923 | Robert Cowham |
Added tests for compressed files gzip and zip. Fix for python2 |
||
| #8 | 23920 | Robert Cowham | Handle multiple log files | ||
| #7 | 23897 | Robert Cowham |
Handle remote journal pulls. Increase size of process key in case of duplicates |
||
| #6 | 23803 | Robert Cowham | Trim newlines - unfortunately all md5s different in tests! | ||
| #5 | 23802 | Robert Cowham |
Handle trigger lapse output when track=1 tableUse has extra field: triggerLapse |
||
| #4 | 23799 | Robert Cowham | Default tests to python3 | ||
| #3 | 23798 | Robert Cowham |
Added explanatory comments. Minor refactoring |
||
| #2 | 23785 | Robert Cowham |
Tweak interface Remove unused module |
||
| #1 | 23765 | Robert Cowham | Moved things down one level to psla dir to make it easier to see what belongs | ||
| //guest/perforce_software/utils/log_analyzer/test_log2sql.py | |||||
| #3 | 23720 | Robert Cowham | Fix failing tests due to NULL values and .p4ignore | ||
| #2 | 23636 | Robert Cowham |
Merge in latests changes to log2sql.py - much faster - better at detecting completed commands and out of order vtrack - full test harness |
||
| #1 | 23635 | Robert Cowham | Branch in tests | ||
| //guest/robert_cowham/perforce/utils/log_analysis/test_log2sql.py | |||||
| #19 | 21796 | Robert Cowham | Handle rmt-FileFetch records which may be duplicates as they don't have 'completed' entries. | ||
| #18 | 21712 | Robert Cowham | Check for correct line number when looking to update for track entries | ||
| #17 | 21708 | Robert Cowham |
Tweak parameters to improve error messages. Make output to stdout optional |
||
| #16 | 21705 | Robert Cowham |
Minor refactoring for clarity. Process all completed records as we go. |
||
| #15 | 21704 | Robert Cowham | A couple more tests for log formats. | ||
| #14 | 21703 | Robert Cowham |
Fix failing python3 test Add new test to ignore error blocks |
||
| #13 | 21702 | Robert Cowham |
Rework algorithm to process blocks at a time, and to avoid the SQL update statements. Risk of increased memory vs improved speed. |
||
| #12 | 21697 | Robert Cowham | Parse rcpSnd/rpcRcv and add to database. | ||
| #11 | 21696 | Robert Cowham | Test for client/locks entry | ||
| #10 | 21694 | Robert Cowham | Handle null values properly | ||
| #9 | 21660 | Robert Cowham | Handle utf8 in python2 strings | ||
| #8 | 21656 | Robert Cowham | Avoid SQL quoting issues with alternative insert statement. | ||
| #7 | 21643 | Robert Cowham | Handle git-fusion entries (strip the json) - for --sql option | ||
| #6 | 21402 | Robert Cowham |
Filter out swarm commands Test for db content in a basic fashion. |
||
| #5 | 21396 | Robert Cowham | Test for update rows. | ||
| #4 | 21393 | Robert Cowham |
Add a swarm test - large record. Improve performance by skipping easy matches early. |
||
| #3 | 21374 | Robert Cowham | Write to sqllite db | ||
| #2 | 21368 | Robert Cowham | Make tests work under python3 | ||
| #1 | 21367 | Robert Cowham | Restructured for unit tests, and first unit tests. | ||