bigpo.ru
добавить свой файл
  1 ... 19 20 21 22 23

def pload(self, conn):

curs = conn.cursor()

curs.execute("select * from %s where pid=%d"%(dbcfg.get('dbOptions', 'PagesTableName'),

self.pid))

page = curs.fetchone()

self.size = page[1]

curs.close()


class Changes(Process):

def __init__(self, conn, traceout=0):

Process.__init__(self)

self.name = 'ChangeProcess'

self.conn = conn

self.traceout = traceout

def life(self):

curs = self.conn.cursor()

t = now()

while 1:

curs.execute("select ctime from %s where ctime>%d order by ctime asc limit 1"%

(dbcfg.get('dbOptions', 'ChangesTableName'), t))

tt = curs.fetchone()

if tt==None:

ChTime = cfg.getint('simOptions', 'ModelTime') + 1

print "finish ", now()

else: ChTime = tt[0] - t

yield hold,self,ChTime

curs.execute("select * from %s where ctime=%d"%

(dbcfg.get('dbOptions','ChangesTableName'), now()))

ev = curs.fetchone()

k=0

while ev:

k+=1

pid = ev[1]

pages[pid].size = ev[4]

prevch = pages[pid].lastch

pages[pid].lastch = ev[3]

if not ((ev[3]==prevch and ev[3] in [1, 2, 3, 6]) or (ev[3]==6 and prevch in [4, 5])):

pages[pid].changed = 1

tmpstat['needfresh'][pid] = 1

if pages[pid].crawled:

pages[pid].crawled = 0

pages[pid].wtime = now()

ev = curs.fetchone()

if k>1: self.trace("%d pages processed"%k)

t = now()

curs.close()

def trace(self,message):

if self.traceout: putfile("at %7.4f %s %s"%(now() , self.name, message))


class Crawl(Process):

def __init__(self, rconn, myrob, traceout=0):

Process.__init__(self)

self.rob = myrob

self.route= []

self.name = 'CrawlProcess'

self.traceout = traceout

self.loadroute(rconn)


def loadroute(self, conn):

curs = conn.cursor()

curs.execute("select * from %s where rob=%d order by pid asc"%

(dbcfg.get('dbOptions','RobRouteTableName'), self.rob))

r = curs.fetchone()

while r:

rr = r[0]

self.route.append(rr)

r = curs.fetchone()

curs.close()


def life(self):

global grv

self.trace("Crawl started")

countpoints = len(self.route)

while 1:

point = 0

while point<=(countpoints-1):

pid = self.route[point]

yield request,self,robot[self.rob]

if pages[pid].lastch not in BadChanges:

size = pages[pid].size

LdTime = grv.randint(cfg.getint('pageOptions', 'MinPageLoadTime'),

cfg.getint('pageOptions', 'MaxPageLoadTime'))

self.trace("begin loading page %s"%pages[pid].pid)

else:

size = 0

LdTime = 0

self.trace("WARNING!!! page type of change is %s"%pages[pid].lastch)

self.trace("loadtime for page ╣%s is %d"%(pages[pid].pid, LdTime))

yield hold, self, LdTime


simvars[self.rob]['SumSizeRob'] += size

self.trace("sumsize for robot ╣%d is %d"%(self.rob, simvars[self.rob]['SumSizeRob']))


if pages[pid].changed:

pages[pid].changed = 0

pages[pid].crawled = 1


del tmpstat['needfresh'][pid]


waitcrawl = now() - pages[pid].wtime

simvars[self.rob]['MaxWaitCrawl'] = max(simvars[self.rob]['MaxWaitCrawl'], waitcrawl)

simvars[self.rob]['MinWaitCrawl'] = min(simvars[self.rob]['MinWaitCrawl'], waitcrawl)

pages[pid].wtime = cfg.getint('simOptions', 'ModelTime')+1

simvars[self.rob]['AvrWaitCrawl'].tally(waitcrawl)

yield release,self,robot[self.rob]


point += 1

simvars[self.rob]['Cycles'] += 1

if simvars[self.rob]['Cycles'].__mod__(cfg.getint('logOptions', 'StepLogCycles'))==0:

putfile("-< Robot %d: %d cycles complete at %s"%\

(self.rob, simvars[self.rob]['Cycles'], now()))


def trace(self,message):

if self.traceout: putfile("at %7.4f %s %s"%(now() , self.name, message))


###[ Statistics classes ]###

class StatVar(Process):

def __init__(self, conn, varname, looktime, myrob = 0):

Process.__init__(self)

self.var = varname

if myrob == 0: self.name = varname

else: self.name = varname + str(myrob)

self.looktime = looktime

self.stime = now()

self.value = None

self.conn = conn

curs = self.conn.cursor()

try: curs.execute("drop table %s"%self.name)

except MySQLdb.Error, msg: print "MySQL error:", msg

curs.execute("create table %s(%s)"%(self.name, dbcfg.get('dbOptions', 'StatTableFieldsSQL')))

curs.close()


def life(self):

td = 0

while 1:

self.setvar()

self.flushstat()

ttime = cfg.getint('simOptions', 'ModelTime') - now()

if ttime
yield hold, self, ttime + td

td = self.looktime

else: yield hold, self, self.looktime


def setvar(self):

self.stime = now()

def flushstat(self):

curs = self.conn.cursor()

curs.execute("insert into %s values('', '%d','%7.2f')"%(self.name, self.stime, self.value))

curs.close()


class Needfresh(StatVar):

def __init__(self, conn):

StatVar.__init__(self, conn, 'needfresh', statvarstime['needfresh'])


def setvar(self):

self.value = len(tmpstat['needfresh'])

StatVar.setvar(self)


class Freshness(StatVar):

def __init__(self, conn):

StatVar.__init__(self, conn, 'freshness', statvarstime['freshness'])


def setvar(self):

k = len(tmpstat['needfresh'])

pcount = cfg.getint('pageOptions', 'PageCount')

self.value = ((pcount - k)*100.0)/pcount

StatVar.setvar(self)


class SumsizeRobs(StatVar):

def __init__(self, conn, myrob):

self.rob = myrob

StatVar.__init__(self, conn, 'sumsizerob', statvarstime['sumsizerobs'], self.rob)


def setvar(self):

k = int(self.rob)

self.value = simvars[k]['SumSizeRob']

StatVar.setvar(self)


class Sumsize(StatVar):

def __init__(self, conn):

StatVar.__init__(self, conn, 'sumsize', statvarstime['sumsize'])


def setvar(self):

self.value = 0

for j in range(cfg.getint('robotOptions','RobotQuantity')):

jj = j+1

self.value += simvars[jj]['SumSizeRob']

StatVar.setvar(self)


###[ Main procedure ]###

def main():

global robot, pages, simvars, pdbconn, chdbconn, statdbconn, routedbconn, grv

putfile("-=[ Simulation Log, started %s ]=-"%strftime("%d.%m.%Y %H:%M:%S", localtime()))

putfile("---[ SimRobot "+__version__+" ]--")

putfile("-< Used %s configuration file"%(wdir+taskdir+'\\ini\\'+taskinifile))


grv = Random(RANDOMSEED)

grv.seed(RANDOMSEED)

putfile("-< Random seed is %s"%(RANDOMSEED))


initialize()


pages = {}

i=1

putfile("-< Begin load pages")

while i<=cfg.getint('pageOptions', 'PageCount'):

p = Page(pdbconn, i)

pages[i] = p

if i.__mod__(cfg.getint('logOptions', 'StepLogLoadPages'))==0:

putfile("-< %s pages loaded at %s"%(i, strftime("%H:%M:%S", localtime())))

i+=1

putfile("-< End load pages")


simvars = {}

robot = {}

webcrawl = {}

ssr = {}


for r in range(cfg.getint('robotOptions', 'RobotQuantity')):

rr = r + 1

simvars[rr] = simvarsdefault.copy()


robot[rr] = Resource(capacity=1, name="R%s"%(rr), unitName="(s)")


webcrawl[rr] = Crawl(routedbconn, rr, 0)

activate(webcrawl[rr],webcrawl[rr].life(),0)


ssr[rr] = SumsizeRobs(statdbconn, rr)

activate(ssr[rr],ssr[rr].life(),0)


putfile("-< Robots created: %s"%(rr))


c = Changes(chdbconn, 0)

activate(c,c.life(),0)


nf = Needfresh(statdbconn)

activate(nf,nf.life(),0)


fr = Freshness(statdbconn)

activate(fr,fr.life(),0)


ss = Sumsize(statdbconn)

activate(ss,ss.life(),0)


putfile("-< Run simulation")

simulate(until=cfg.getint('simOptions', 'ModelTime'))

putfile("-< End simulation")

putfile("--[ Statistics ]-------------------------------------------")

ssize = 0

for r in range(cfg.getint('robotOptions','RobotQuantity')):

rr = r + 1

putfile("-< Robot: %d"%(rr))

simvarsr = simvars[rr].copy()

for key in simvarsr:

try:

simk = simvarsr[key]

simt = simk.mean()

except (AttributeError, TypeError):

putfile("-< %s = %d"%(key, simvarsr[key]))

if key == 'SumSizeRob': ssize += simvarsr[key]

else:

putfile("-< %s = %4.1f"%(simk.name, simk.mean()))

if simk.name == 'SumSizeRob': ssize += simk.mean()

putfile("-< All robots: ")

putfile("-< SumSize = %d"%(ssize))

putfile("--[ Statistics ]-------------------------------------------")

print "\n"


putfile("---[ SimRobot "+__version__+" ]--")

putfile("-=[ Simulation Log, finished %s ]=-"%strftime("%d.%m.%Y %H:%M:%S", localtime()))

putfile("\n")

flog.close()

pdbconn.close()

chdbconn.close()

statdbconn.close()

routedbconn.close()


###[ Main program ]###

if __name__ == '__main__':

main()


Приложение 6.


Листинг модуля SimSensor.py

### SimSensor.py, by Ilya A. Zemskov (zemskov@univer.omsk.su)


__version__ = '$ Revision: 0.9.2 $ Date: 2003/07/24 11:18:00 $'


from __future__ import generators

from time import localtime, strftime

from random import Random, expovariate, uniform, randint, random

from SimPy.Simulation import *

from SimPy.Monitor import Monitor

import MySQLdb

from ConfigParser import *

from os import getcwd

import sys


###[ Simulation variables ]###

BadChanges = [1,2,3]

WasntChange = [6]

comini = "..\\ini\\common.ini"

dbini = "..\\ini\\db.ini"

taskinifile = "test.ini"


###[ Special variables ]###

try: taskdir = sys.argv[1]

except IndexError, msg: sys.exit("Error: missing parameter - name of task directory")


comcfg = ConfigParser()

comcfg.read(comini)

wdir = comcfg.get('Options','WorkDirectory')


dbcfg = ConfigParser()

dbcfg.read(dbini)


cfg = ConfigParser()

cfg.read(wdir+taskdir+'\\ini\\'+taskinifile)


flog = file(cfg.get('logOptions','LogDirectory') + cfg.get('logOptions','LogFileName'), "a+")


pdbconn = MySQLdb.Connect(dbcfg.get('dbOptions', 'Host'), dbcfg.get('dbOptions', 'Login'),

dbcfg.get('dbOptions', 'Password'), cfg.get('dbOptions', 'PagesDBName'))


chdbconn = MySQLdb.Connect(dbcfg.get('dbOptions', 'Host'), dbcfg.get('dbOptions', 'Login'),

dbcfg.get('dbOptions', 'Password'), cfg.get('dbOptions', 'ChangesDBName'))


qdbconn = MySQLdb.Connect(dbcfg.get('dbOptions', 'Host'), dbcfg.get('dbOptions', 'Login'),

dbcfg.get('dbOptions', 'Password'), cfg.get('dbOptions', 'QueriesDBName'))


statdbconn = MySQLdb.Connect(dbcfg.get('dbOptions', 'Host'), dbcfg.get('dbOptions', 'Login'),

dbcfg.get('dbOptions', 'Password'), cfg.get('dbOptions', 'StatDBName'))


TRACETOFILE = cfg.getint('logOptions','LogTraceToFile')

RANDOMSEED = cfg.getint('simOptions','RandomSeed')


simvarsdefault = {\

'SumSize' :0,\

'MinWaitCrawl' :cfg.getint('simOptions', 'ModelTime'),\

'MaxWaitCrawl' :0,\

'AvrWaitCrawl' :Monitor('AvrWaitCrawl'),\

'MaxProcessing':0,\

'NowProcessing':0}


statvarstime = {\

'needfresh' :5000,\

'freshness' :5000,\

'sumsize' :10000,\

'nowprocessing':5000}


tmpstat = {\

'needfresh' :{},\

'freshness' :0,\

'wtime' :0}


###[ General procedures ]###

def putfile(st):

if TRACETOFILE:

flog.write(st+"\n")

flog.flush()

else: print st


###[ Simulation classes ]###

class Page:

def __init__(self, conn, pid):

self.pid = pid

self.lastch = cfg.getint('pageOptions', 'DefaultPageChange')

self.size = 0

self.changed = cfg.getint('pageOptions', 'DefaultPageChanged')

self.crawled = cfg.getint('pageOptions', 'DefaultPageCrawled')

self.wtime = cfg.getint('simOptions', 'ModelTime')

self.pload(conn)

def pload(self, conn):

curs = conn.cursor()

curs.execute("select * from %s where pid=%d"%(dbcfg.get('dbOptions', 'PagesTableName'),

self.pid))

page = curs.fetchone()

self.size = page[1]

curs.close()


class Changes(Process):

def __init__(self, conn, traceout=0):

Process.__init__(self)

self.name = 'ChangeProcess'

self.conn = conn

self.traceout = traceout

def life(self):

curs = self.conn.cursor()

t = now()

while 1:

curs.execute("select ctime from %s where ctime>%d order by ctime asc limit 1"%

(dbcfg.get('dbOptions', 'ChangesTableName'), t))

tt = curs.fetchone()

if tt==None:

ChTime = cfg.getint('simOptions', 'ModelTime')+1

print "finish ", now()

else: ChTime = tt[0]-t

yield hold, self, ChTime

curs.execute("select * from %s where ctime=%d"%

(dbcfg.get('dbOptions', 'ChangesTableName'), now()))

ev = curs.fetchone()

k=0

while ev:

k+=1

pid = ev[1]

pages[pid].size = ev[4]

prevch = pages[pid].lastch

pages[pid].lastch = ev[3]

if not ((ev[3]==prevch and ev[3] in [1, 2, 3, 6]) or (ev[3]==6 and prevch in [4, 5])):

pages[pid].changed = 1

tmpstat['needfresh'][pid] = 1

if pages[pid].crawled:

pages[pid].crawled = 0

pages[pid].wtime = now()

ev = curs.fetchone()

if k>1: self.trace("%d pages processed"%k)

t = now()

curs.close()


<< предыдущая страница   следующая страница >>