Never been to DZone Snippets before?

Snippets is a public source code repository. Easily build up your personal collection of code snippets, categorize them with tags / keywords, and share them with the world

About this user

Peter Cooperx http://www.petercooper.co.uk/

« Newer Snippets
Older Snippets »
Showing 1-5 of 5 total  RSS 

Fix Content-Length header on UTF8 with HTTP::Daemon in Perl

   1  
   2  It seems that HTTP/Daemon.pm is bogus when it calculate
   3  the header 'Content-Length' when the data contains UTF8 data .
   4  
   5  In attachement a patch , to calculate the length in bytes
   6  of the data .
   7  
   8  -- 
   9       ____________________________________________________________
  10      / Erwan MAS                                                 /\
  11     | mailto:[EMAIL PROTECTED]                                   |_/
  12  ___|________________________________________________________   |
  13  \___________________________________________________________\__/
  14  
  15  --- Daemon.pm.orig      2004-12-11 16:13:22.000000000 +0100
  16  +++ Daemon.pm   2006-05-02 22:53:33.660393022 +0200
  17  @@ -436,7 +436,7 @@
  18              }
  19          }
  20          elsif (length($content)) {
  21  -           $res->header("Content-Length" => length($content));
  22  +           $res->header("Content-Length" => bytes::length($content));
  23          }
  24          else {
  25              $self->force_last_request;

Using an HTTP proxy with Universal Feed Parser

without using http_proxy environment variable..

   1  import urllib2, feedparser
   2  
   3  proxy = urllib2.ProxyHandler( {"http":"http://your.proxy.here:8080/"} )
   4  d = feedparser.parse('http://feedparser.org/docs/examples/atom10.xml', handlers = [proxy])

Fetch a URL in Ruby

   1  require 'uri'
   2  require 'net/http'
   3  url = "http://www.whatever.com/whatever.txt"
   4  r = Net::HTTP.get_response(URI.parse(url).host, URI.parse(url).path)


r.code = 200 | 404 | 500, etc
r.body = *text of page*

View returned HTTP header on a request

   1  curl -I http://www.google.com/

Pre-forking HTTP daemon in Perl

Version 2.. it had issues with zombie / defunct processes on Linux. This is significantly more stable, and I'm currently serving 300,000+ requests a day on something based off of this:

   1  #!/usr/bin/perl
   2  
   3  # Basic pre-forking HTTP daemon - version 2
   4  # By Peter Cooper - http://www.petercooper.co.uk/
   5  #
   6  # Inspiration and various rehashed snippetsof code from the Perl 
   7  # cfdaemon engine - http://perl-cfd.sourceforge.net/
   8  #
   9  # You can switch out HTTP::Daemon and make it a pre-forking daemonized 
  10  # 'anything' if you wish..
  11  
  12  use HTTP::Daemon;
  13  use HTTP::Status;
  14  use CGI;
  15  use POSIX;
  16  
  17  my $totalChildren = 15;				# Number of listening children to keep alive
  18  my $childLifetime = 10;			# Let each child serve up to this many requests
  19  my $logFile = "/tmp/daemon.log";	# Log requests and errors to this file
  20  my %children;							# Store pids of children
  21  my $children = 0;						# Store number of currently active children
  22  
  23  &daemonize;								# Daemonize the parent
  24  
  25  my $d = HTTP::Daemon->new( LocalPort => 1981, LocalAddr => '127.0.0.1', Reuse => 1, Timeout => 180 ) || die "Cannot create socket: $!\n";
  26  
  27  warn ("master is ", $d->url);
  28  
  29  &spawnChildren;
  30  &keepTicking;
  31  exit;
  32  
  33  
  34  # spawnChildren - initial process to spawn the right number of children
  35  
  36  sub spawnChildren {
  37  	for (1..$totalChildren) {
  38  		&newChild();
  39  	}
  40  }
  41  
  42  
  43  # keepTicking - a never ending loop for the parent process which just monitors
  44  # dying children and generates new ones
  45  
  46  sub keepTicking {
  47  	while ( 1 ) {
  48  		sleep;
  49  	  	for (my $i = $children; $i < $totalChildren; $i++ ) {
  50  		  &newChild();
  51  		}
  52  	};
  53  }
  54  
  55  
  56  # newChild - a forked child process that actually does some work
  57  
  58  sub newChild {
  59  	my $pid;
  60  	my $sigset = POSIX::SigSet->new(SIGINT);				# Delay any interruptions!
  61     sigprocmask(SIG_BLOCK, $sigset) or die "Can't block SIGINT for fork: $!";
  62     die "Cannot fork child: $!\n" unless defined ($pid = fork);
  63  	if ($pid) {
  64  		$children{$pid} = 1;										# Report a child is using this pid
  65  		$children++;												# Increase the child count
  66  		warn "forked new child, we now have $children children";
  67  		return;														# Head back to wait around
  68  	}
  69  	
  70  	my $i;
  71  	while ($i < $childLifetime) {				# Loop for $childLifetime requests
  72  		$i++;
  73  		my $c = $d->accept or last;							# Accept a request, or if timed out.. die early
  74  		$c->autoflush(1);
  75  		logMessage ("connect:". $c->peerhost . "\n");	# We've accepted a connection!
  76       	my $r = $c->get_request(1) or last;					# Get the request. If it fails, die early
  77  
  78  		# Insert your own logic code here. The request is in $r
  79  		# What we do here is check if the method is not GET, if so.. send back a 403.
  80  
  81  		my $url = $r->url;
  82  		$url =~ s/^\///g;
  83  
  84       	if ($r->method ne 'GET') { 
  85  			$c->send_error(RC_FORBIDDEN); 
  86  			logMessage ($c->peerhost . " made weird request\n"); 
  87  			redo;
  88  		}
  89  		
  90  		my $response = HTTP::Response->new(200);			# Put together a response
  91  		logMessage ($c->peerhost . " " . $d->url . $url . "\n");	
  92  		$response->content("<html><body>The daemon works! This child has served $i requests.</body></html>");
  93  #				$response->content("document.write('OK $i<br \/>');");
  94  		$response->header("Content-Type" => "text/html");
  95  		$c->send_response($response);							# Send back a basic response
  96  		
  97  		logMessage ("disconnect:" . $c->peerhost . " - ct[$i]\n");		# Log the end of the request
  98        $c->close;
  99  	}
 100  	
 101  	warn "child terminated after $i requests";
 102  	exit;
 103  }
 104  
 105  
 106  # REAPER - a reaper of dead children/zombies with exit codes to spare
 107  
 108  sub REAPER {                            
 109  	my $stiff;
 110  	while (($stiff = waitpid(-1, &WNOHANG)) > 0) {
 111  		warn ("child $stiff terminated -- status $?");
 112  		$children--;
 113  		$children{$stiff};
 114  	}
 115  	$SIG{CHLD} = \&REAPER;
 116  }        
 117  
 118  # daemonize - daemonize the parent/control app
 119  
 120  sub daemonize {
 121  	my $pid = fork;												# Fork off the main process
 122  	defined ($pid) or die "Cannot start daemon: $!"; 	# If no PID is defined, the daemon failed to start
 123  	print "Parent daemon running.\n" if $pid;				# If we have a PID, the parent daemonized okay
 124  	exit if $pid;													# Return control to the user
 125  
 126     # Now we're a daemonized parent process!
 127  
 128  	POSIX::setsid();												# Become a session leader
 129  
 130  	close (STDOUT);												# Close file handles to detach from any terminals
 131  	close (STDIN);
 132  	close (STDERR);
 133  
 134  	# Set up signals we want to catch. Let's log warnings, fatal errors, and catch hangups and dying children
 135  
 136  	$SIG{__WARN__} = sub {
 137  			&logMessage ("NOTE! " . join(" ", @_));
 138  	};
 139  	
 140  	$SIG{__DIE__} = sub { 
 141  		&logMessage ("FATAL! " . join(" ", @_));
 142  		exit;
 143  	};
 144  
 145  	$SIG{HUP} = $SIG{INT} = $SIG{TERM} = sub {			# Any sort of death trigger results in instant death of all
 146  	  my $sig = shift;
 147  	  $SIG{$sig} = 'IGNORE';
 148  	  kill 'INT' => keys %children;
 149  	  die "killed by $sig\n";
 150  	  exit;
 151  	};	
 152  	
 153  	$SIG{CHLD} = \&REAPER;
 154  }
 155  
 156  # logMessage - append messages to a log file. messy, but it works for now.
 157  
 158  sub logMessage {
 159  	my $message = shift;
 160  	(my $sec, my $min, my $hour, my $mday, my $mon, my $year) = gmtime();
 161  	$mon++;
 162  	$mon = sprintf("%0.2d", $mon);
 163  	$mday = sprintf("%0.2d", $mday);
 164  	$hour = sprintf("%0.2d", $hour);
 165  	$min = sprintf("%0.2d", $min);
 166  	$sec = sprintf("%0.2d", $sec);
 167  	$year += 1900;
 168  	my $time = qq{$year/$mon/$mday $hour:$min:$sec};
 169  	open (FH, ">>" . $logFile);
 170  	print FH $time . " - " . $message;
 171  	close (FH);
 172  }
« Newer Snippets
Older Snippets »
Showing 1-5 of 5 total  RSS