Efficient actions counter using tornado IOLoop.add_timeout
Redis used as temporary storage and sqlite3 as persistent storage for counters.
pip install tornado==4.1
pip install redis
Create sqlite table:
(.env)nanvel-air:tornado_counter nanvel$ sqlite3 actions.sqlite3
SQLite version 3.8.5 2014-08-15 22:37:57
Enter ".help" for usage hints.
sqlite> CREATE TABLE counters(action TEXT PRIMARY KEY, count INTEGER DEFAULT 0);
import datetime
import logging
import os
import sqlite3
import redis
from tornado import web, ioloop, gen, options
logger = logging.getLogger(__name__)
class ActionHandler(web.RequestHandler):
ALLOWED_ACTIONS = ['open_browser', 'open_new_tab', 'enter_search_term', 'scroll', 'click_on_link']
REDIS_COUNTER_KEY = 'actions:counter:{action}'
REDIS_TASK_KEY = 'actions:task:{action}'
UPDATE_PERIOD = 30 # seconds
def inc_count(self, action):
key = self.REDIS_COUNTER_KEY.format(action=action)
counter_key = self.REDIS_COUNTER_KEY.format(action=action)
task_key = self.REDIS_TASK_KEY.format(action=action)
count, delted_counter, deleted_task = self.application.redis.pipeline(
).get(counter_key).delete(counter_key).delete(task_key).execute()
if not count:
return
with sqlite3.connect(self.application.db_path) as connection:
cursor = connection.cursor()
result = cursor.executescript("INSERT OR REPLACE INTO counters(action, count) VALUES ('{action}', COALESCE((SELECT count + 1 FROM counters WHERE action = '{action}'), 1));".format(
count=count, action=action))
logger.warning('Saved to sqlite3.')
def post(self):
action = self.get_body_argument('action')
if action in self.ALLOWED_ACTIONS:
counter_key = self.REDIS_COUNTER_KEY.format(action=action)
task_key = self.REDIS_TASK_KEY.format(action=action)
count, task_exists = self.application.redis.pipeline().incr(counter_key, 1).get(task_key).execute()
if not task_exists:
ioloop.IOLoop.instance().add_timeout(
deadline=datetime.timedelta(seconds=self.UPDATE_PERIOD),
callback=self.inc_count,
action=action)
self.application.redis.setex(task_key, self.UPDATE_PERIOD * 2, 1)
logger.warning('Incremented.')
self.write('Ok')
class CounterApplication(web.Application):
def __init__(self, *args, **kwargs):
handlers = [(r'/action', ActionHandler)]
kwargs['debug'] = True
self.db_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'counters.sqlite3')
self.redis = redis.StrictRedis(host='localhost')
super(CounterApplication, self).__init__(handlers, *args, **kwargs)
if __name__ == "__main__":
options.parse_command_line()
CounterApplication().listen(5000)
ioloop.IOLoop.instance().start()
# curl --data "action=open_browser" http://localhost:5000/action
# [W 150308 20:27:00 app:46] Incremented.
# [I 150308 20:27:00 web:1825] 200 POST /action (::1) 7.90ms
# [W 150308 20:27:01 app:46] Incremented.
# [I 150308 20:27:01 web:1825] 200 POST /action (::1) 1.74ms
# [W 150308 20:27:02 app:46] Incremented.
# [I 150308 20:27:02 web:1825] 200 POST /action (::1) 1.81ms
# [W 150308 20:27:05 app:32] Saved to sqlite3.
# [W 150308 20:27:10 app:46] Incremented.
# [I 150308 20:27:10 web:1825] 200 POST /action (::1) 1.56ms
# [W 150308 20:27:11 app:46] Incremented.
# [I 150308 20:27:11 web:1825] 200 POST /action (::1) 1.29ms
# [W 150308 20:27:12 app:46] Incremented.
# [I 150308 20:27:12 web:1825] 200 POST /action (::1) 1.38ms
# [W 150308 20:27:12 app:46] Incremented.
# [I 150308 20:27:12 web:1825] 200 POST /action (::1) 1.38ms
# [W 150308 20:27:13 app:46] Incremented.
# [I 150308 20:27:13 web:1825] 200 POST /action (::1) 1.29ms
# [W 150308 20:27:14 app:46] Incremented.
# [I 150308 20:27:14 web:1825] 200 POST /action (::1) 1.72ms
# [W 150308 20:27:15 app:32] Saved to sqlite3.
(.env)nanvel-air:tornado_counter nanvel$ sqlite3 counters.sqlite3
SQLite version 3.8.5 2014-08-15 22:37:57
Enter ".help" for usage hints.
sqlite> select * from counters;
open_new_tab|3
open_browser|21
Licensed under CC BY-SA 3.0