Never been to DZone Snippets before?

Snippets is a public source code repository. Easily build up your personal collection of code snippets, categorize them with tags / keywords, and share them with the world

About this user

Vsevolod Balashov

« Newer Snippets
Older Snippets »
Showing 1-7 of 7 total  RSS 

Thread Pool and Threaded Python decorators

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim:ts=4:sw=4:et
# (c) 2007 under terms of LGPL v 2.1
# by Vsevolod S. Balashov <vsevolod@balashov.name> 

from threading import Thread

def threaded(func):
    def proxy(*args, **kwargs):
        thread = Thread(target=func, args=args, kwargs=kwargs)
        thread.start()
        return thread
    return proxy

from Queue import Queue

class Pool(Queue):
    def __init__(self, maxsize):
        assert maxsize > 0, 'maxsize > 0 required for Pool class'
        Queue.__init__(self, maxsize)
        for i in range(maxsize):
            thread = Thread(target = self._worker)
            thread.setDaemon(True)
            thread.start()

    def _worker(self):
        while True:
            try:
                func, args, kwargs = self.get()
                func(*args, **kwargs)
            except:
                self.task_done()
                self.join()
                raise
            self.task_done()

    def addJob(self, func, *args, **kwargs):
        self.put((func, args, kwargs))

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_value, traceback):
        self.join()

def threadpool(pool):
    assert pool.__class__ == Pool, 'threadpool decorator require a Pool object'
    def decorator(func):
        def proxy(*args, **kwargs):
            pool.put((func, args, kwargs))
            return pool
        return proxy
    return decorator


example usage

from threadpool import *

from time import sleep
from random import random

pool = Pool(3)
    
@threadpool(pool)
def test_threadpool(i):
    print 'threadpool %i enter' % i
    sleep(random())
    print 'threadpool %i exit' % i

print 'threadpool example'
for i in range(6):
    test_threadpool(i)
pool.join()
print 'done'
print ''

@threaded
def test_threaded(i):
    print 'threaded %i enter' % i
    sleep(random())
    print 'threaded %i exit' % i

print 'threaded example'
for i in range(5):
    test_threaded(i)
print 'done'


result output

threadpool example
threadpool 0 enter
threadpool 1 enter
threadpool 2 enter
threadpool 0 exit
threadpool 3 enter
threadpool 1 exit
threadpool 4 enter
threadpool 2 exit
threadpool 5 enter
threadpool 3 exit
threadpool 5 exit
threadpool 4 exit
done

threaded example
done
threaded 0 enter
threaded 1 enter
threaded 2 enter
threaded 3 enter
threaded 4 enter
threaded 1 exit
threaded 2 exit
threaded 3 exit
threaded 0 exit
threaded 4 exit

Storm ORM by Canonical in WSGI enabled applications

#!/usr/bin/env python
# vim:ts=4:sw=4:et
# (c) 2007 Vsevolod Balashov <vsevolod@balashov.name> under terms of LGPL 2.1
# use Storm ORM <https://storm.canonical.com> in WSGI enabled applications

"""WSGI middleware for Storm.

This is the database access inteface for WSGI enabled (PEP 333) web applications.

Pylons framework example:

in wsgiapp.py

from storm.database import create_database
from middlestorm import MiddleStorm
...

# CUSTOM MIDDLEWARE HERE
app = MiddleStorm(app, create_database(config['app_conf']['sqlalchemy.default.uri']))

in controller:

class DemoController(BaseController):
    def index(self):
        store = request.environ['storm.store']
        ...
"""

from storm.database import Database
from storm.store import Store
from threading import local

__all__ = ["MiddleStorm"]
__author__ = "Vsevolod Balashov <http://vsevolod.balashov.name>"
__version__ = "0.1"

class SingleConn(Database):
    """Database proxy class.
    Share single database connection between all Store object instances in threads.
    If you have readonly database or not use transactions why not?
    """
    def __init__(self, database):
        self._database = database
        self._connection = database.connect()
    def connect(self):
        return self._connection

class MiddleStorm(object):
    """WSGI middleware.
    Add Store object instance in environ['storm.store']. Each thread contains own instance.
    """

    def __init__(self, app, database, single = False):
        """Create WSGI middleware.
        @param app: up level application or middleware.
        @param database: instance of Database returned create_database.
        @param: single: use single database connection in all threads. 
        """
        assert isinstance(database, Database), \
            'database must be subclass of storm.database.Database'
        if single:
            self._database = SingleConn(database)
        else:
            self._database = database
        self._app = app
        self._local = local()

    def __call__(self, environ, start_response):
        try:
            environ['storm.store']  = self._local.store
        except AttributeError:
            environ['storm.store']  = \
                self._local.__dict__.setdefault('store', Store(self._database))
        return self._app(environ, start_response)

Ruby port of range2cidr. Convert IP ranges to set of CIDR.

See rfc1878 for more detail about CIDR

#!/usr/bin/env ruby
# -*- coding: utf-8 -*-
# $Hg: range_cidr.rb,v af9566d89389 2007-04-12 20:28 +0400 $
# (C) 2007 under terms of LGPL v2.1
# by Vsevolod S. Balashov <vsevolod@balashov.name>
#
# backported from perl code
# http://www.irbs.net/internet/postfix/0401/att-3032/cidr_range.pl.gz

def range_cidr(first, last, &block)
  if first < last
    idx1 = 32
    idx1 -= 1 while first[idx1] == last[idx1]
    prefix = first >> idx1+1 << idx1+1

    idx2 = 0
    idx2 += 1 while idx2 <= idx1 and first[idx2] == 0 and last[idx2] == 1

    if idx2 <= idx1
      range_cidr(first, prefix | 2**idx1-1, &block)
      range_cidr(prefix | 1 << idx1, last, &block)
    else
      yield prefix, 32-idx2
    end
  else
    yield first, 32
  end
end


Example Usage

#!/usr/bin/env ruby
# $Hg: range2cidr.rb,v 2142c33ada8b 2007-04-11 23:14 +0400 $
# (C) 2007 under terms of GPL v2
# by Vsevolod S. Balashov <vsevolod@balashov.name>
#
# example usage of range_cidr.rb

require 'lib/range_cidr'
require 'ipaddr'
require 'socket'

if __FILE__ == $0
  if ARGV.size == 2
    range_cidr(IPAddr.new(ARGV[0]).to_i, IPAddr.new(ARGV[1]).to_i) { |subnet, mask|
      puts "#{IPAddr.new(subnet, Socket::AF_INET).to_s}/#{mask}"  }
  else
    puts "usage: range2cidr <first_ip> <last_ip>"
    puts "example: range2cidr 192.168.1.0 192.168.2.255"
  end
end


Type in terminal and look result

$ ruby range2cidr.rb 192.168.1.0 192.168.2.255
192.168.1.0/24
192.168.2.0/24


I hate perl.

Google PageRank Ruby Checker

#!/usr/bin/env ruby
# -*- coding: utf-8 -*-
# $Id: google-pr.rb,v b205c14e4ef5 2007-01-15 13:53 +0300 $
# (C) 2006-2007 under terms of LGPL v2.1 
# by Vsevolod S. Balashov <vsevolod@balashov.name>
# based on 3rd party code snippets (see comments)

require 'uri'
require 'open-uri'

module SEO

  # http://blog.outer-court.com/archive/2004_06_27_index.html#108834386239051706
  class GooglePR

    def initialize(uri)
      @uri = uri
    end

    M=0x100000000 # modulo for unsigned int 32bit(4byte)

    def m1(a,b,c,d)
      (((a+(M-b)+(M-c))%M)^(d%M))%M # mix/power mod
    end

    def i2c(i)
      [i&0xff, i>>8&0xff, i>>16&0xff, i>>24&0xff]
    end

    def c2i(s,k=0)
      ((s[k+3].to_i*0x100+s[k+2].to_i)*0x100+s[k+1].to_i)*0x100+s[k].to_i
    end

    def mix(a,b,c)
      a = a%M; b = b%M; c = c%M
      a = m1(a,b,c, c >> 13); b = m1(b,c,a, a <<  8); c = m1(c,a,b, b >> 13)
      a = m1(a,b,c, c >> 12); b = m1(b,c,a, a << 16); c = m1(c,a,b, b >>  5)
      a = m1(a,b,c, c >>  3); b = m1(b,c,a, a << 10); c = m1(c,a,b, b >> 15)
      [a, b, c]
    end

    def old_cn(iurl = 'info:' + @uri)
      a = 0x9E3779B9; b = 0x9E3779B9; c = 0xE6359A60
      len = iurl.size 
      k = 0
      while (len >= k + 12) do
        a += c2i(iurl,k); b += c2i(iurl,k+4); c += c2i(iurl,k+8)
        a, b, c = mix(a, b, c)
        k = k + 12
      end
      a += c2i(iurl,k); b += c2i(iurl,k+4); c += (c2i(iurl,k+8) << 8) + len
      a,b,c = mix(a,b,c)
      return c
    end

    def cn
      ch = old_cn
      ch = ((ch/7) << 2) | ((ch-(ch/13).floor*13)&7)
      new_url = []
      20.times { i2c(ch).each { |i| new_url << i }; ch -= 9 }
      ('6' + old_cn(new_url).to_s).to_i
    end

    def request_uri
      # http://www.bigbold.com/snippets/posts/show/1260 + _ -> %5F
      "http://toolbarqueries.google.com/search?client=navclient-auto&hl=en&ch=#{cn}&ie=UTF-8&oe=UTF-8&features=Rank&q=info:#{URI.escape(@uri, /[^-.!~*'()a-zA-Z\d]/)}"
    end
 
    def page_rank(uri = @uri)
      @uri = uri if uri != @uri
      open(request_uri) { |f| return $1.to_i if f.string =~ /Rank_1:\d:(\d+)/ }
      nil
    end

    private :m1, :i2c, :c2i, :mix, :old_cn
    attr_accessor :uri
  end

end

if __FILE__ == $0 and 1 == ARGV.size
  puts SEO::GooglePR.new(ARGV[0]).page_rank
end

nowww!

set redirect from www.example.com to example.com for nginx web server

server {
    listen       80;
    server_name  example.com www.example.com;
    if ( $host = www.example.com ) {
        rewrite ^\/(.*)$ http://example.com/$1 permanent;
    }

    .....
}

backup subversion repository over email

#!/bin/bash
#
# $Id: repodiff 3 2006-09-21 18:48:39Z sevkin $
#
# subversion repository incremental backup over e-mail
#
# (c) 2006 Vsevolod Balashov under terms of GNU GPL v.2 or later

SVNROOT=/var/svn
EMAIL=your@email.here
STORE=`mktemp -d`
GPGCRYPT=n

for REPO in `ls $SVNROOT`; do 
	REPOPATH=$SVNROOT/$REPO;
	if [ -r $REPOPATH/youngest ]; then
		LATEST=`cat $REPOPATH/youngest`
		YOUNGEST=`svnlook youngest $REPOPATH`
		if [ $LATEST -lt $YOUNGEST ]; then
			svnadmin dump $REPOPATH --incremental -r $LATEST:$YOUNGEST >$STORE/$REPO 2>/dev/null
		fi
	else
		svnadmin dump $REPOPATH --incremental >$STORE/$REPO 2>/dev/null
	fi 
	echo $YOUNGEST >$REPOPATH/youngest 
done

if [ `ls $STORE | wc -w` -gt 0 ]; then
	BACKUP=repodiff_`date -u +%Y%m%d%H%M%S`.tar.bz2
	ATTACH=$STORE/../$BACKUP
	tar  -C $STORE -cjf $ATTACH .
	if [ $GPGCRYPT = y ]; then
		gpg -e -r $EMAIL $ATTACH
		ATTACH=$ATTACH.gpg
	fi
	echo "." | mutt -c $EMAIL -a $ATTACH -s "repository incremental backup"
	rm -f $ATTACH
fi

rm -rf $STORE

Subversion service for low-load (personal?) sources repository

Create separate user and insert svnserve into inetd.

# useradd -g root -s /bin/false -d /dev/null -c "SubVersion Daemon" svnserve
# mkdir /var/svn
# chown -R svnserve /var/svn
# update-inetd --add 'svn\tstream\ttcp\tnowait\tsvnserve\t/usr/sbin/tcpd\t/usr/bin/svnserve --inetd --root /var/svn'


/var/svn - root of repository.
update-inetd - standart tool in debian and ubuntu linux distros

You must run svnadmin as svnserve user for manage your repository

$ sudo sudo -u svnserve svnadmin <command>


« Newer Snippets
Older Snippets »
Showing 1-7 of 7 total  RSS