DZone Snippets: parse code

ARGV Parser

Black_H (Black_H) — Sun, 03 Feb 2008 20:24:59 GMT

This function parse ARGV and return a string.
See exemples for more informations :

// Go : http://blackh.badfile.net/wordz/

Function :



  def options(param)

  

	i = 0

		ARGV.each  { |valeur|

		

    		if (valeur == '-' + param.to_s)

				return ARGV[i+1]

			elseif (valeur != '-' + param.to_s)

				return false

			end

		i += 1

		}

		

   end

Usage :

// cmd> ruby test.rb -o foo





	out =  self.options('o')



	if (out != false and out.empty? == false)

                   puts out # print -> foo

	end

Regex-based parsing technique

ntk () — Thu, 06 Dec 2007 15:43:11 GMT





#!/usr/local/bin/ruby -w



# Regular expressions and strings with embedded objects

# From: http://t-a-w.blogspot.com/2007/06/regular-expressions-and-strings-with.html

# Author: Tomasz Węgrzanowski

# License: 

# Creative Commons License, http://creativecommons.org/licenses/by-sa/3.0/

# GNU Free Documentation License, http://en.wikipedia.org/wiki/GNU_Free_Documentation_License





def hash_or_die(kw)

  Hash.new{|ht,k| raise "Unknown key: #{k}"}.merge(kw)

end



def parse(data)

  esc = hash_or_die "\\" => "A", "\"" => "B", "n" => "C", "'" => "D"

  rev_esc = hash_or_die "A" => "\\", 'B' => "\"", "C" => "n", "D" => "'"

  data = data.gsub(/\\(.)/) {"\x00" + esc[$1]}

  strs = []

  data = data.gsub(/('[^']*')/) { # '

    strs << $1

    "\x01<#{strs.size-1}>"

  }

  records = []

  data.scan(/\((.*?)\)/) {

    records << $1.split(/,/).map{|field|

      field.gsub(/\x01<(\d+)>/) {

        strs[$1.to_i]}.gsub(/\x00(.)/){ rev_esc[$1]

      }

    }

  }

  records

end



def sql_str_unquote(str)

  str =~ /\A'(.*)'\Z/ or raise "SQL string format is wrong: #{str}"

  $1.gsub(/\\(.)/) {$1}

end





=begin



page_fn = Dir["plwiki-*-page.sql"].sort[-1]

externallinks_fn = Dir["plwiki-*-externallinks.sql"].sort[-1]



pages = {}



File.open(page_fn).each{|line|

  next unless line =~ /\AINSERT INTO `page` VALUES (.*)\Z/

  parse($1).each{|id,ns,title,*stuff|

    next unless ns == "0"

    title = sql_str_unquote(title)

    pages[id] = title

  }

}



File.open(externallinks_fn).each{|line|

  next unless line =~ /\AINSERT INTO `externallinks` VALUES (.*)\Z/

  parse($1).each{|from,to,index|

    title = pages[from]

    next unless title

    to = sql_str_unquote(to)

    next unless to =~ /\Ahttp:\/\//

    puts "#{title}\t#{to}"

  }

}



=end





sql_dump = <<-EOS



INSERT INTO `page` VALUES (1,0,'Astronomia','',1800,0,0,0.600461925007833,'20070601091320',8076762,8584,0), (2,0,'AWK','',329,0,0,0.487812640599732,'20070530195555',8058046,4265,0), (4,0,'Alergologia','',108,0,0,0.580574716050713,'20070520093413',7912844,292,0), ...

INSERT INTO `page` VALUES (14880,0,'Dźwignica_linotorowa','',26,0,0,0.597327036408081,'20060814072401',4282357,727,0), (14881,0,'Urządzenia_transportowe','',91,0,0,0.176666489966834,'20070527090143',2976610,1041,0), ...



EOS





pages = {}



sql_dump.each{|line|

  next unless line =~ /\AINSERT INTO `page` VALUES (.*)\Z/

  parse($1).each{|id,ns,title,*stuff|

    next unless ns == "0"

    title = sql_str_unquote(title)

    pages[id] = title

  }

}





p pages



=begin



sql_dump.each{|line|

  next unless line =~ /\AINSERT INTO `externallinks` VALUES (.*)\Z/

  parse($1).each{|from,to,index|

    title = pages[from]

    next unless title

    to = sql_str_unquote(to)

    next unless to =~ /\Ahttp:\/\//

    puts "#{title}\t#{to}"

  }

}



=end





#-----------------





require 'pp'



lisp_code = '(a (b c) (d (e) f g) (((h))))'

nodes = []



lisp_code.gsub!(/([a-z]+)/) {

  nodes << [:atom, $1]

  "<#{nodes.size-1}>"

}



#p nodes



lisp_code.gsub!(/\s/,"")

#puts lisp_code



true while lisp_code.gsub!(/\(((?:<\d+>)*)\)/) {

  #p nodes

  nodes << [:app, *$1.scan(/<(\d+)>/).map{|x,| nodes[x.to_i]}]

  "<#{nodes.size-1}>"

}

lisp_code =~ /<(\d+)>/



#puts

#p nodes

#puts



pp nodes[$1.to_i]



# Output:

#  [:app,

#  [:atom, "a"],

#  [:app, [:atom, "b"], [:atom, "c"]],

#  [:app, [:atom, "d"], [:app, [:atom, "e"]], [:atom, "f"], [:atom, "g"]],

#  [:app, [:app, [:app, [:atom, "h"]]]]]





#------------------





math_code = '(2 + 2 * 2) / ((2 + 2) * 2)'

nodes = []



math_code.gsub!(/(\d+)/) {

  nodes << $1.to_i

  "<#{nodes.size-1}>"

}

math_code.gsub!(/\s/,"")



until math_code =~ /\A<(\d+)>\Z/

  next if math_code.gsub!(/\((<\d+>)\)/) { $1 }

  next if math_code.gsub!(/<(\d+)>([\*\/])<(\d+)>/) {

    nodes << [$2, nodes[$1.to_i], nodes[$3.to_i]]

    "<#{nodes.size-1}>"

  }

  next if math_code.gsub!(/<(\d+)>([\+\-])<(\d+)>/) {

    nodes << [$2, nodes[$1.to_i], nodes[$3.to_i]]

    "<#{nodes.size-1}>"

  }

end



pp nodes[$1.to_i]



# Output:

# ["/", ["+", 2, ["*", 2, 2]], ["*", ["+", 2, 2], 2]]

xml parsing using content handler

mwaschkowski (Mark Waschkowski) — Tue, 30 Oct 2007 18:04:26 GMT

// description of your code here
given an xml:





	managePermissions

	

		

			role1

			role2

			role3

		

		

			roleA

			roleB

		

	

	

		xyz

will parse into permission map



static Map> getPermissionMap(String xml){

    final Map> permissionMap = new HashMap>();

    

    //handler will produce a permission map based on xml

    ContentHandler allowDenyXmlHandler = new DefaultHandler() {

      private String permissionType;

      private List roleList = new ArrayList();

      private Stack nodes = new Stack();

      private StringBuilder roleBuffer = new StringBuilder();

      

      /**

       * Get type of permission that is defined in type attribute

       */

      public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

        nodes.push(qName.trim());

        //get type of permission

        if(nodes.peek().equals(PERMISSION)){

          permissionType = attributes.getValue(PERMISSION_TYPE);

        }

      }



      /**

       * Get role data for specified role

       */

      public void characters(char ch[], int start, int length) throws SAXException {

        if(nodes.peek().equals(ROLE)){

          roleBuffer.append(ch, start, length);  

        }

      }



      /**

       * Populate role list if role end tag, or populate permission map if permission end tag

       */

      public void endElement(String uri, String localName, String qName) throws SAXException {

        String nodeName = nodes.pop();

        if(nodeName.equals(ROLE)){

          roleList.add(roleBuffer.toString());

          roleBuffer.setLength(0);  

        }else if(nodeName.equals(PERMISSION)){

          List tempList = new ArrayList();

          tempList.addAll(roleList);

          permissionMap.put(permissionType, tempList);

          roleList.clear();

        }

      }



      public void endDocument() throws SAXException {

        // sanity check

        if (nodes.size() > 0) logger.error("Node stack is not empty !!!");

      }

    };

    

    try {

      SAXParser parser = null;

      // get SAXParser

      synchronized (XMLParserHelper.saxParserFactory) {

        parser = XMLParserHelper.saxParserFactory.newSAXParser();

      }

      // Set custom content handler

      parser.getXMLReader().setContentHandler(allowDenyXmlHandler);

      // parse xml

      parser.getXMLReader().parse(new InputSource(new StringReader(xml)));



    } catch (Exception e) {

      logger.error("[DLA00002] Unable to process permissions from XML {" + xml + "}", e);

    }

    

    return permissionMap;

  }

Parse Exim log file with SpamAssassin score

Wallace (Wallace) — Sat, 29 Sep 2007 08:50:51 GMT



grep '\[Spam score: [1-9]' /var/log/exim/main.log | awk '{print $3}' | xargs -t -i grep {} /var/log/exim/main.log > /var/log/exim/spam_score.log



grep 'reporter\.pl' /var/log/exim/main.log | awk '{print $3}' | xargs -t -i grep {} /var/log/exim/main.log > /var/log/exim/spam_reporter.pl.log

simple backup PHP application (php files + mysql db)

chenull (Sayid Munawar) — Fri, 14 Sep 2007 00:15:12 GMT

// backup a folder contains all .php files and it's mysql database. a backup.ini file is required for storing mysql root login and what folder to be backed up.



#!/bin/sh



########################################

# simple sctipt for daily / hourly backup of PHP app + Mysql DB

# Copyleft (c) Sayid Munawar. chenull@yahoo.com

# LICENSE: anyone can copy / modify / distribute. whatever lah

#

# example of workhours backup (8 am - 5 pm. Mon - Fri)

# 0 8-17 * * 1-5 /home/backup/backup.sh

#

# example of simple dayly backup ( 6 pm. Mon - Fri)

# 0 18 * * 1-5 /home/backup/backup.sh

########################################



########################################

# example of backup.ini. REMOVE ALL leading '#'s

# file must be chmod 600

#[main]

#target = /home/backup/storage

#mysql_user = root   ; root can connect to any db

#mysql_pass = secret ; mysql root password

#

#[hukum]

#path = /home/hukum

#mysql_db = hukum

#

#[phpmyadmin]

#path = /home/phpmyadmin

#mysql_db = test

########################################



PATH=$PATH:/usr/bin:/bin:/usr/local/bin



inifile=`dirname $0`/backup.ini



test ! -r $inifile && echo "ERROR! backup.ini not found nor readable" && exit 1



#test apakah backup.ini bisa dibaca orang lain

echo -n `stat -c '%a' $inifile` | grep -q '[0-7]00' >/dev/null 2>&1

test $? -gt 0 && echo "ERROR! $inifile can be read by others" && exit 2

#test `stat -c '%U' $inifile` != 'root' && echo "ERROR! $inifile ownernya bukan root" && exit 3



# function to parse ini file (backup.ini)

#

# $1 -> file.ini

# $2 -> section

_parse_ini () {

    if [ -z "$1" ] || [ -z "$2" ]; then return 0; fi

    eval `cat $1 | \

       sed -e 's/[[:space:]]*\=[[:space:]]*/=/g' \

           -e 's/;.*$//' \

           -e 's/[[:space:]]*$//' \

           -e 's/^[[:space:]]*//' \

           -e "s/^\(.*\)=\([^\"']*\)$/\1=\"\2\"/" | \

       sed -n -e "/^\[$2\]/,/^\s*\[/{/^[^;].*\=.*/p;}"`

}



_parse_ini $inifile main



# coba konek 

mysql -u $mysql_user  -p${mysql_pass} -e '' > /dev/null 2>&1

test $? -gt 0 && echo "ERROR! Failed to connect to mysql" && exit 4



# bikin target kalo blum ada

test ! -d $target && mkdir -p $target



hari=`date +%A`

jam=`date +%H`



# get pwd

cwd=`pwd`



# looping

for section in `cat $inifile | grep '^\[' | grep -v main | sed 's/\[//' | sed 's/\]//'`; do

  outdir=$target/$section/$hari/$jam

  echo Backing up ${section} to $outdir...

  # parse .ini

  _parse_ini $inifile $section

  # cek dulu

  test ! -d $path && echo "Warning: $path not found.  Backup process of $section skipped" && echo && continue

  mysql -u $mysql_user  -p${mysql_pass} -e '' $mysql_db > /dev/null 2>&1

  test $? -gt 0 && echo "Warning: Database $mysql_db not found.  Backup process of $section skipped" && echo && continue



  test ! -d $outdir && mkdir -p $outdir

  cd $path

  tar -czpf $outdir/$section.tar.gz .

  cd $cwd

  mysqldump -Q -u $mysql_user -p${mysql_pass} $mysql_db | gzip > $outdir/${mysql_db}.sql.gz

  ln -fs $outdir/$section.tar.gz $target/$section/latest.tar.gz

  ln -fs $outdir/${mysql_db}.sql.gz $target/$section/latest.sql.gz

done

CSV parsing regex

ntk () — Sat, 18 Aug 2007 17:17:23 GMT

The regular expression is taken from Raimond Brookman, Regex fun with CSV.
For a good general CSV overview see The Comma Separated Value (CSV) File Format.
A complete Ruby CSV parsing library is FasterCSV (sudo gem install fastercsv).





csv_data = <<-EOS



fname,lname,age,salary

nancy,davolio,33,$30000

erin,borakova,28,$25250

tony,raphael,35,$28700



"Date","Pupil","Grade"

"25 May","Bloggs, Fred","C"

"25 May","Doe, Jane","B"

"15 July","Bloggs, Fred","D"



123456789,"Carr, Lisa",100000.00

444556666,"Barr, Clark",87000.00

777227878,"Parr, Jack",123000.00

998877665,"Charr, Lee",123000.00



Conference room 1, "John,  

Please bring the M. Mathers file for review  

-J.L.

"

10/18/2002,...



John,Doe,120 jefferson st.,Riverside, NJ, 08075

Jack,McGinnis,220 hobo Av.,Phila, PA,09119

"John ""Da Man""",Repici,120 Jefferson St.,Riverside, NJ,08075

Stephen,Tyler,"7452 Terrace ""At the Plaza"" road",SomeTown,SD, 91234

,Blankman,,SomeTown, SD, 00298

"Joan ""the bone"", Anne",Jet,"9th, at Terrace plc",Desert City,CO,00123



XXXX,D,3-May-02,83.01,83.58,71.13,78.04,9645300

XXXX,D,2-May-02,82.47,85.76,82.05,83.84,7210000,

XXXX,D,1-May-02,86.80,90.83,81.74,85.50,14253300



"1997",car model,E350

1997,car model,E350,"  Super luxurious truck    "

1997,car model,E350,"Go get one now

they are going fast"

1997,car model,E350,"Super ""luxurious"" truck"

1997,car model,E350,"Super, luxurious truck"



1997,car model,E350,"ac, abs, moon",3000.00

1999, car model,"Venture ""Extended Edition""",,4900.00,

1996, car model,Old Car,"BEYOND REPAIR!

air, moon roof, loaded",4799.00



This,is,a test,CSV, file," from ""http://lorance.freeshell.org/csv/test.csv""."

It contains,"quoted text",and,numbers 1234,5678

It also has,"quoted text with an embedded quote""<- right there"

Then there are a few,,blank fields like these here ->,,,

A quoted blank field,"",<- there.

A quoted blank field with newline,"\n",<- there.

This next one causes an error if newline handling is turned off.

"There is a newline here ->

<- and it should be processed correctly."

ABCD

"And here,,, is an""Error - no"

"And here,,, is an"Error - yes

"And here,,, is an",Error - no



1,2,3

ab,"c,d","e""f", "g"",""","h

jk",kl



"aaa","bbb","ccc"

zzz,yyy,xxx

"aaa","b

bb","ccc"

zzz,yyy,xxx



"aaa","b""bb","ccc"



EOS







csv_data.split(/(,|\r\n|\n|\r)(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/m).each do |csv|

#csv_data.split(/[,\n\r]+(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))/m).each do |csv|



   next if csv.empty?



   csv = csv.strip



   if csv =~ /\A(".*[^"]|[^"].*")\z/m then     # examples: csv => "ab\nc"def  or  abc"de\nf"

       puts

       puts "Error:" 

       p csv 

       puts csv[/\A./mu], csv[/.\z/mu] 

       #puts csv[0..0], csv[-1..-1] 

       puts

       next

   end



   if csv =~ /\A".*"\z/m then csv.gsub!(/\A"(.*)"\z/m, '\1') end  # remove double-quotes at string beginning & end

   if csv =~ /""/m then csv.gsub!(/""/m, '"') end                 # remove a double-quote from double double-quotes



   p csv



end

Fix for camping multipart parsing issue.

NetherBen (Ben) — Thu, 26 Jul 2007 01:53:58 GMT

// description of your code here
I wrote a nice description of this issue, but snippets ate it.

In short, this parses complicated form names into nice hashes, i.e. name[asd][asd] for multipart forms.



module Camping

module Base

 def initialize(r, e, m) #:nodoc:

      e = H[e.to_hash]

      @status, @method, @env, @headers, @root = 200, m.downcase, e,

          {'Content-Type'=>'text/html'}, e.SCRIPT_NAME.sub(/\/$/,'')

      @k = C.kp(e.HTTP_COOKIE)

      qs = C.qs_parse(e.QUERY_STRING)

      @in = r

      todo = []

      if %r|\Amultipart/form-data.*boundary=\"?([^\";,]+)|n.match(e.CONTENT_TYPE)

        b = /(?:\r?\n|\A)#{Regexp::quote("--#$1")}(?:--)?\r$/

        until @in.eof?

          fh=H[]

          for l in @in

            case l

            when Z: break

            when /^Content-Disposition: form-data;/

              fh.u H[*$'.scan(/(?:\s(\w+)="([^"]+)")/).flatten]

            when /^Content-Type: (.+?)(\r$|\Z)/m

              puts "=> fh[type] = #$1"

              fh[:type] = $1

            end

          end

          fn=fh[:name]

          o=if fh[:filename]

            o=fh[:tempfile]=Tempfile.new(:C)

            o.binmode

          else

            fh=""

          end

          while l=@in.read(16384)

            if l=~b

              o<<$`.chomp

              @in.seek(-$'.size,IO::SEEK_CUR)

              break

            end

            o<< l

          end

          if(fn)

            qs.merge!(C.qs_parse(fn + "=1"))

            parts = fn.split(/\[/)

            line = ""

            parts.each do |pp|

              pp = pp.gsub(/\]/,"")

              line += "['#{pp}']"

            end

            #p "line is:"

            #p line

            #p fh

            todo << ["qs#{line}", fh]

            #eval("qs#{line} = fh")

            #p qs

          end

          fh[:tempfile].rewind if fh.is_a?H

        end

      elsif @method == "post"

        qs.merge!(C.qs_parse(@in.read))

      end

      #post process vars

      todo.each do |n|

        eval("#{n.first} = n.last")

      end

      #p "END QUERY"

      #p qs

      #exit

      @cookies, @input = @k.dup, qs.dup

    end





end 

end

// insert code here..

Making md5sum.exe Work with Paths in Python

minitotoro (Natalie) — Wed, 13 Jun 2007 23:10:02 GMT

On Windows systems, md5sum.exe is a nice little program to generate MD5 sums. However, whoever created this application forgot to add the ability to recognize paths in the file given to md5sum.exe. For example, say you want to md5 a file called test.txt.

C:\test>dir
Directory of C:\test

06/13/2007 03:52 PM .
06/13/2007 03:52 PM ..
06/13/2007 03:52 PM 0 test.txt
1 File(s) 0 bytes

C:\test>md5sum test.txt
d41d8cd98f00b204e9800998ecf8427e *test.txt

Groovy, cool...

However, if you try the same thing from some other directory (where text.txt does not live) you get the following:

C:\>md5sum c:/test/test.txt
md5sum: test.txt: No such file or directory

Sucky, eh?
You could always find a different program to do md5sum. Or, if you don't want to do this, you can spawn a pipe to "cmd", navigate to the correct directory, and then run md5sum.

Here's the code in python to do just that.



# define your file name and path

file_name = "afile.txt"

file_path = "c:\\afolder"



# setup the md5sum command (assuming md5sum.exe location is in PATH)

md5_cmd = "md5sum \"" + file_path "\\" + file_name + "\"\n"



# open the command shell

fromchild, tochild = popen2.popen4("cmd")



#push the directory change and md5sum commands to the shell 

tochild.write("c:\nchdir " + my_path + "\n" + md5_cmd + "exit\n")

tochild.close()



#get the output from the shell session

out = fromchild.read()



#split the output so that we may extract the md5sum

output = string.split(out)



#grab the md5sum  

md5sum_local = ""

for item in output:

    matchmd5local = re.match("([0-9a-fA-F]{32})",item)

    if matchmd5local:

        md5sum_local = matchmd5local.groups()

        md5sum_local = md5sum_local[0]



if md5sum_local:

    print_status("MD5: Local checksum = " + md5sum_local + "\n")

else:

    print_status("ERROR: could not obtain local md5sum for " + my_file + "\n")

Ruby CSV to XML Converter

jnorris (Jason Norris) — Mon, 19 Mar 2007 19:17:05 GMT

This code will take an input CSV file and output XML. IT was easy to write, but I haven't found anything out there to do this. The first line of the CSV file should contain the element names.



#!/usr/bin/ruby



require 'csv'



print "CSV file to read: "

input_file = gets.chomp



print "File to write XML to: "

output_file = gets.chomp



print "What to call each record: "

record_name = gets.chomp



csv = CSV::parse(File.open(input_file) {|f| f.read} )

fields = csv.shift



puts "Writing XML..."



File.open(output_file, 'w') do |f|

  f.puts ''

  f.puts ''

  csv.each do |record|

    f.puts " <#{record_name}>"

    for i in 0..(fields.length - 1)

      f.puts "  <#{fields[i]}>#{record[i]}"

    end

    f.puts " "

  end

  f.puts ''

end # End file block - close file



puts "Contents of #{input_file} written as XML to #{output_file}."

named-fields?

gregg.irwin (Gregg Irwin) — Thu, 16 Nov 2006 05:16:23 GMT



	; This can't guarantee that a record is actually a series of

	; named fields; it can only tell if it *isn't*.

	named-fields?: func [rec [block!]] [

		parse rec [some [word! any-type!]]

	]