Package flumotion :: Package component :: Package misc :: Package httpserver :: Module cachestats
[hide private]

Source Code for Module flumotion.component.misc.httpserver.cachestats

  1  # -*- Mode: Python; test-case-name: -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  import time 
 23   
 24  from twisted.internet import reactor 
 25   
 26  from flumotion.common import log 
 27   
 28   
 29  # Filter out requests that read less than a block for average values 
 30  MIN_REQUEST_SIZE = 64 * 1024 + 1 
 31  # Statistics update period 
 32  STATS_UPDATE_PERIOD = 10 
 33   
 34  CACHE_MISS = 0 
 35  CACHE_HIT = 1 
 36  TEMP_HIT = 2 
 37   
 38   
39 -class RequestStatistics(object):
40
41 - def __init__(self, cacheStats):
42 self._stats = cacheStats 43 self._outdated = False 44 self._size = 0L 45 self._status = None 46 self.bytesReadFromSource = 0L 47 self.bytesReadFromCache = 0L
48
49 - def getBytesRead(self):
50 return self.bytesReadFromSource + self.bytesReadFromCache
51 bytesRead = property(getBytesRead) 52
53 - def getCacheReadRatio(self):
54 total = self.bytesRead 55 if total == 0: 56 return 0.0 57 return float(self.bytesReadFromCache) / total
58 cacheReadRatio = property(getCacheReadRatio) 59
60 - def onStarted(self, size, cacheStatus):
61 cs = self._stats 62 self._size = size 63 if cacheStatus == CACHE_HIT: 64 self._status = "cache-hit" 65 cs.cacheHitCount += 1 66 elif cacheStatus == TEMP_HIT: 67 cs.cacheHitCount += 1 68 cs.tempHitCount += 1 69 self._status = "temp-hit" 70 elif cacheStatus == CACHE_MISS: 71 cs.cacheMissCount += 1 72 if self._outdated: 73 self._status = "cache-outdate" 74 else: 75 self._status = "cache-miss" 76 cs._set("cache-hit-count", cs.cacheHitCount) 77 cs._set("temp-hit-count", cs.tempHitCount) 78 cs._set("cache-miss-count", cs.cacheMissCount)
79
80 - def onCacheOutdated(self):
81 self._outdated = True 82 cs = self._stats 83 cs.cacheOutdateCount += 1 84 cs._set("cache-outdate-count", cs.cacheOutdateCount)
85
86 - def onBytesRead(self, fromSource, fromCache, correction):
87 cs = self._stats 88 self.bytesReadFromSource += fromSource + correction 89 self.bytesReadFromCache += fromCache - correction 90 cs.bytesReadFromSource += fromSource + correction 91 cs.bytesReadFromCache += fromCache - correction
92
93 - def onClosed(self):
94 pass
95
96 - def getLogFields(self):
97 """ 98 Provide the following log fields: 99 cache-status: value can be 'cache-miss', 'cache-outdate', 100 'cache-hit', or 'temp-hit' 101 cache-read: how many bytes where read from the cache for 102 this resource. the difference from resource-read 103 was read from the source file (network file system?) 104 105 The proportion read from cache and from source are adjusted 106 to take into account the file copy. It's done by remembering 107 how many bytes are copied at session level. 108 """ 109 return {"cache-status": self._status, 110 "cache-read": self.bytesReadFromCache}
111 112
113 -class CacheStatistics(object):
114 115 _updater = None 116 _callId = None 117
118 - def __init__(self):
119 # For cache usage 120 self._cacheUsage = 0 121 self._cacheUsageRatio = 0.0 122 # For cache statistics 123 self.cacheHitCount = 0 124 self.tempHitCount = 0 125 self.cacheMissCount = 0 126 self.cacheOutdateCount = 0 127 self.cleanupCount = 0 128 # For real file reading statistics 129 self.bytesReadFromSource = 0L 130 self.bytesReadFromCache = 0L 131 # File copying statistics 132 self.totalCopyCount = 0 133 self.currentCopyCount = 0 134 self.finishedCopyCount = 0 135 self.cancelledCopyCount = 0 136 self.bytesCopied = 0L 137 self._copyRatios = 0.0
138
139 - def startUpdates(self, updater):
140 self._updater = updater 141 if updater and (self._callId is None): 142 self._set("cache-usage-estimation", self._cacheUsage) 143 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio) 144 self._set("cleanup-count", self.cleanupCount) 145 self._set("last-cleanup-time", time.time()) 146 self._set("current-copy-count", self.currentCopyCount) 147 self._set("finished-copy-count", self.finishedCopyCount) 148 self._set("cancelled-copy-count", self.cancelledCopyCount) 149 self._set("mean-copy-ratio", self.meanCopyRatio) 150 self._set("mean-bytes-copied", self.meanBytesCopied) 151 self._update()
152
153 - def stopUpdates(self):
154 self._updater = None 155 if self._callId is not None: 156 self._callId.cancel() 157 self._callId = None
158
159 - def getCacheReadRatio(self):
160 total = self.bytesReadFromSource + self.bytesReadFromCache 161 if total == 0: 162 return 0 163 return float(self.bytesReadFromCache) / total
164 cacheReadRatio = property(getCacheReadRatio) 165
166 - def getMeanBytesCopied(self):
167 if self.finishedCopyCount == 0: 168 return 0 169 return self.bytesCopied / self.finishedCopyCount
170 meanBytesCopied = property(getMeanBytesCopied) 171
172 - def getMeanCopyRatio(self):
173 if self.finishedCopyCount == 0: 174 return 0 175 return self._copyRatios / self.finishedCopyCount
176 meanCopyRatio = property(getMeanCopyRatio) 177
178 - def onEstimateCacheUsage(self, usage, max):
179 self._cacheUsage = usage 180 self._cacheUsageRatio = float(usage) / max 181 self._set("cache-usage-estimation", self._cacheUsage) 182 self._set("cache-usage-ratio-estimation", self._cacheUsageRatio)
183
184 - def onCleanup(self):
185 self.cleanupCount += 1 186 self._set("cleanup-count", self.cleanupCount) 187 self._set("last-cleanup-time", time.time())
188
189 - def onCopyStarted(self):
190 self.currentCopyCount += 1 191 self.totalCopyCount += 1 192 self._set("current-copy-count", self.currentCopyCount)
193
194 - def onCopyCancelled(self, size, copied):
195 self.currentCopyCount -= 1 196 self.finishedCopyCount += 1 197 self.cancelledCopyCount += 1 198 self.bytesCopied += copied 199 self._copyRatios += float(copied) / size 200 self._set("current-copy-count", self.currentCopyCount) 201 self._set("finished-copy-count", self.finishedCopyCount) 202 self._set("cancelled-copy-count", self.cancelledCopyCount) 203 self._set("mean-copy-ratio", self.meanCopyRatio) 204 self._set("mean-bytes-copied", self.meanBytesCopied)
205
206 - def onCopyFinished(self, size):
207 self.currentCopyCount -= 1 208 self.finishedCopyCount += 1 209 self.bytesCopied += size 210 self._copyRatios += 1.0 211 self._set("current-copy-count", self.currentCopyCount) 212 self._set("finished-copy-count", self.finishedCopyCount) 213 self._set("mean-copy-ratio", self.meanCopyRatio) 214 self._set("mean-bytes-copied", self.meanBytesCopied)
215
216 - def _set(self, key, value):
217 if self._updater is not None: 218 self._updater.update(key, value)
219
220 - def _update(self):
221 self._set("cache-read-ratio", self.cacheReadRatio) 222 self._logStatsLine() 223 self._callId = reactor.callLater(STATS_UPDATE_PERIOD, self._update)
224
225 - def _logStatsLine(self):
226 """ 227 Statistic fields names: 228 CRR: Cache Read Ratio 229 CMC: Cache Miss Count 230 CHC: Cache Hit Count 231 THC: Temp Hit Count 232 COC: Cache Outdate Count 233 CCC: Cache Cleanup Count 234 CCU: Cache Current Usage 235 CUR: Cache Usage Ratio 236 PTC: coPy Total Count 237 PCC: coPy Current Count 238 PAC: coPy cAncellation Count 239 MCS: Mean Copy Size 240 MCR: Mean Copy Ratio 241 """ 242 log.debug("stats-local-cache", 243 "CRR: %.4f; CMC: %d; CHC: %d; THC: %d; COC: %d; " 244 "CCC: %d; CCU: %d; CUR: %.5f; " 245 "PTC: %d; PCC: %d; PAC: %d; MCS: %d; MCR: %.4f", 246 self.cacheReadRatio, self.cacheMissCount, 247 self.cacheHitCount, self.tempHitCount, 248 self.cacheOutdateCount, self.cleanupCount, 249 self._cacheUsage, self._cacheUsageRatio, 250 self.totalCopyCount, self.currentCopyCount, 251 self.cancelledCopyCount, self.meanBytesCopied, 252 self.meanCopyRatio)
253