http://snippets.dzone.com/posts Sun, 18 May 2008 00:52:37 GMT DZone Snippets: quoted code http://snippets.dzone.com/posts/show/5459 An exercise in string processing and regexp matching, inspired by <a href="http://mikenaberezny.com/2008/04/28/parsing-quoted-strings-in-ruby/">Parsing Quoted Strings in Ruby</a> and <a href="http://pivots.pivotallabs.com/users/nick/blog/articles/409-stupid-ruby-quoting-tricks/">Stupid Ruby Quoting Tricks</a>. <br /> <br /><code> <br />#!/usr/local/bin/ruby -w <br /> <br /># some input examples <br />str = 'foo "bar baz" qux' <br />str = 'foo "bar baz " "bar baz" " bar baz" "bar "klr mre" " " \' "abc" \' baz " qux' <br />str = '" \' \' " \n " " \' \' "" foo \'ttt sss\' "bar "qqq zzz" baz" "added term" qux " \' \' " yyy xxx' <br />str = '"""frickin \'#{bar}\'"""' <br />str = '"" "frickin chicken " #{bar}""""' <br />str = '"""frickin "#{bar}""""' <br />str = '"a "b c" "d "e" f g" """h""""' # cf. http://snippets.dzone.com/posts/show/4852 <br /> <br /># escaped quotes <br />str = '\"' <br />str = "\\\"" <br />str = '\\\'' <br />str = "\\'" <br /> <br /># special cases <br />str = '"G","H I"' <br />str = '"G","H I""G","H I"' <br /> <br />str = '"abc""def"' <br />str = '"""a""b"' <br />str = '"abc""def""abc""def""abc""def"' <br />str = '"a"\'\'"b"' <br /> <br />str = "\"abc'vv'tt\"'klt'" <br /> <br />str = "abc,def,\"efg,hij\",klm,nop,\"qrstuv\",wxyz" <br />str = "abc,def,\"efg,hij\",klm, 'nop, \"qrstuv\",wxyz,mmm '" <br />str = "abc,def,\"efg,hij\",klm, \"nop, 'qrstuv',wxyz,mmm \"" <br /> <br /> <br />puts <br />puts "input string: #{str}" <br />puts "str.inspect : #{str.inspect}" <br />puts <br /> <br />num_of_chars1 = str.count('a-zA-Z_0-9', "^\000ds") <br /> <br />error_code = 0 # in case of a parsing error Shellwords will be used instead of regex1 & regex2 <br />str2 = str.clone <br /> <br /># encode escaped quotes <br />str = str.gsub(/\\"|\\'/) { |m| m =~ /^\\"$/ ? "\000d\000" : "\000s\000" } <br /> <br />dq_count = str.count('"') <br />sq_count = str.count("'") <br /> <br />if dq_count % 2 != 0 && sq_count % 2 != 0 <br /> raise ArgumentError, "\e[1modd number of single & double quotes\e[m in: #{str}\nsq_count: #{sq_count}\ndq_count: #{dq_count}\n" <br />elsif dq_count % 2 != 0 <br /> raise ArgumentError, "\e[1modd number of double quotes\e[m in: #{str}\ndq_count: #{dq_count}\n" <br />elsif sq_count % 2 != 0 <br /> raise ArgumentError, "\e[1modd number of single quotes\e[m in: #{str}\nsq_count: #{sq_count}\n" <br />end <br /> <br /># regex1 separates substrings that contain quotes from substrings that do not contain quotes <br />regex1 = %r{[^"']+|["'].*?["'](?!.*["'])}m <br /> <br /># example <br />#"abc 'quote1' pjk 'quote2' xyz".scan(regex1) { |m| puts m } <br /> <br /> <br />regex2 = %r{ <br /># experimental: special cases <br />\s*["'][^"']+["'][[:punct:]]["'][^"']+["']| # special case: xxx "ab c","def g" yyy <br />\s*["'][^"']+["']{2,}[^"']+["']| # special case: xxx "abc""def" yyy <br />\s"[^"]+"| # special case: xxx "abc 'def' ghi" <br />\s'[^']+'| # special case: xxx 'abc "def" ghi' <br />\s*["']\S+["']| # special case: "abc'vv'tt"'klt' <br /> <br />\s'\s| # xxx ' yyy <br />\s"\s| # xxx " yyy <br />\s''\s| # xxx '' yyy <br />\s""\s| # xxx "" yyy <br />\s'\s+'\s| # xxx ' ' yyy <br />\s"\s+"\s| # xxx " " yyy <br />\s"\s[^"]+\s"\s| # xxx " abc " yyy <br />\s'\s[^']+\s'\s| # xxx ' abc ' yyy <br />\s["']["']+(?=[^"'\s])| # :qoblock: xxx "'""'abc yyy <br />[^"'\s]["']["']+(?=\s)| # :qcblock: xxx abc"'""' yyy <br />\s""+| # :dqoblock: xxx """abc yyy <br />\s''+| # :sqoblock: xxx '''abc yyy <br />[^"]""+| # :dqcblock: xxx abc"" yyy <br />[^']''+| # :sqcblock: xxx abc'' yyy <br />\s["'](?=[^"'\s])| # :dqo or :sqo: xxx "abc yyy or xxx 'abc yyy <br />[^"'\s]["'](?=\s)| # :dqc or :sqc: xxx abc" yyy or xxx abc' yyy <br />[^"']+[^"'\s](?=\s) # no quotes at all <br />}mx <br /> <br /> <br />=begin <br /> <br />There are different kinds of quotes matched by regex2 below. They include: <br /> <br />- :sqo (single quote open) <br />- :sqc (single quote close) <br />- :sqoblock (single quote open block) <br />- :sqcblock (single quote close block) <br /> <br />- :dqo (double quote open) <br />- :dqc (double quote close) <br />- :dqoblock (double quote open block) <br />- :dqcblock (double quote close block) <br /> <br />- :qoblock (quote open block) <br />- :qcblock (quote close block) <br /> <br />=end <br /> <br /> <br />ret = [] <br /> <br />str.scan(regex1) do |s| <br /> <br /> if s !~ /\A["']/ <br /> <br /> #puts "s1: #{s}" <br /> #puts "s1.inspect: #{s.inspect}" <br /> <br /> s.split(/\s+/m).each { |t| ret << t unless t.empty? } <br /> <br /> else <br /> <br /> #puts "s2: #{s}" <br /> #puts "s2.inspect: #{s.inspect}" <br /> <br /> open_quotes = 0 <br /> close_quotes = 0 <br /> ar = [] <br /> <br /> # add spaces to simplify regex2 matching <br /> s = "\x20" << s << "\x20" <br /> s.gsub!(/\x20/, "\x20\x20") <br /> <br /> <br /> s.scan(regex2) do |m| <br /> <br /> # get the index of the quote <br /> # + 1 for leading space or non-space <br /> # $` is the prematch string <br /> <br /> index = $`.length + 1 <br /> <br /> post_match = $' <br /> <br /> #puts <br /> #puts "index: #{index}" <br /> #puts "m: #{m.inspect}" <br /> #puts "m.length: #{m.length}" <br /> #puts "open_quotes: #{open_quotes}\nclose_quotes: #{close_quotes}" <br /> #puts "ret: #{ret.inspect}" <br /> #puts "ar: #{ar.inspect}" <br /> #puts <br /> <br /> <br /> if m =~ /\A\s''\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << '' <br /> next <br /> <br /> elsif m =~ /\A\s""\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << "" <br /> next <br /> <br /> # example: xxx "ab c","def g" yyy <br /> elsif open_quotes.zero? && close_quotes.zero? && m =~ /\A\s*["'][^"']+["'][[:punct:]]["'][^"']+["']\z/ && m.count('"') % 2 == 0 && m.count("'") % 2 == 0 <br /> <br /> m = m.gsub(/\x20\x20/, "\x20") <br /> # cf. http://henrik.nyh.se/2008/03/flickr-style-tag-splitting-in-ruby <br /> m = m.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? } <br /> #m = m.split(/"(.+?)"|'(.+?)'|\s+/).reject {|sm| sm.empty? } <br /> #m = m.split(/"(.+?)"|'(.+?)'|([[:punct:]])|\s+/).reject {|sm| sm.empty? } <br /> ret.concat(m) <br /> next <br /> <br /> # example: xxx "abc""def" yyy <br /> elsif open_quotes.zero? && close_quotes.zero? && m =~ /\A\s*["'][^"']+["']{2,}[^"']+["']\z/ && m.count('"') % 2 == 0 && m.count("'") % 2 == 0 <br /> <br /> m = m.gsub(/\x20\x20/, "\x20") <br /> m = m.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? } <br /> #m = m.split(/"(.+?)"|'(.+?)'|\s+/).reject {|sm| sm.empty? } <br /> ret.concat(m) <br /> next <br /> <br /> <br /> elsif open_quotes.zero? && close_quotes.zero? && m =~ /\A\s"[^"]+"\z/ && m.count('"') % 2 == 0 && m.count("'") % 2 == 0 <br /> ret.concat(m.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? }) <br /> next <br /> <br /> elsif open_quotes.zero? && close_quotes.zero? && m =~ /\A\s'[^']+'\z/ && m.count('"') % 2 == 0 && m.count("'") % 2 == 0 <br /> ret.concat(m.split(/'(.+?)'|\s+/).reject {|sm| sm.empty? }) <br /> next <br /> <br /> elsif open_quotes.zero? && close_quotes.zero? && m =~ /\A\s*["']\S+["']\z/ && m.count('"') % 2 == 0 && m.count("'") % 2 == 0 <br /> ret.concat(m.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? }) <br /> next <br /> <br /> <br /> elsif m =~ /\A\s"\s[^"]+\s"\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << m.gsub(/\x20\x20/, "\x20").strip[1..-2] <br /> next <br /> <br /> elsif m =~ /\A\s'\s[^']+\s"\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << m.gsub(/\x20\x20/, "\x20").strip[1..-2] <br /> next <br /> <br /> elsif m =~ /\A\s'\s+'\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << m.gsub(/\x20\x20/, "\x20").strip[1..-2] <br /> next <br /> <br /> elsif m =~ /\A\s"\s+"\s\z/ <br /> <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> ret << m.gsub(/\x20\x20/, "\x20").strip[1..-2] <br /> next <br /> <br /> <br /> elsif m =~ /\A\s""+\z/ <br /> <br /> l = m.strip.length <br /> ar << [:dqoblock, index, l] <br /> old_open_quotes = open_quotes <br /> open_quotes += l <br /> <br /> if close_quotes == 0 && old_open_quotes == 0 && open_quotes % 2 == 0 && post_match !~ /"/ <br /> ret << m[2..-2] <br /> open_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> <br /> elsif m =~ /\A\s''+\z/ <br /> <br /> l = m.strip.length <br /> ar << [:sqoblock, index, l] <br /> old_open_quotes = open_quotes <br /> open_quotes += l <br /> <br /> if close_quotes == 0 && old_open_quotes == 0 && open_quotes % 2 == 0 && post_match !~ /'/ <br /> ret << m[2..-2] <br /> open_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> <br /> elsif m =~ /\A[^"]""+\z/ <br /> <br /> l = m[1..-1].strip.length <br /> ar << [:dqcblock, index+l-1, l] # index+l-1 is the index of the last closing quote: ''"'['] <br /> old_close_quotes = close_quotes <br /> close_quotes += l <br /> <br /> if open_quotes == 0 && old_close_quotes == 0 && close_quotes % 2 == 0 && post_match !~ /"/ <br /> ret << m[2..-2] <br /> close_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> elsif m =~ /\A[^']''+\z/ <br /> <br /> l = m[1..-1].strip.length <br /> ar << [:sqcblock, index+l-1, l] <br /> old_close_quotes = close_quotes <br /> close_quotes += l <br /> <br /> if open_quotes == 0 && old_close_quotes == 0 && close_quotes % 2 == 0 && post_match !~ /'/ <br /> ret << m[2..-2] <br /> close_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> <br /> elsif m =~ /\A\s'\z/ <br /> <br /> ar << [:sqo, index, 1] <br /> open_quotes += 1 <br /> <br /> elsif m =~ /\A\S'\z/ <br /> <br /> ar << [:sqc, index, 1] <br /> close_quotes += 1 <br /> <br /> elsif m =~ /\A\s"\z/ <br /> <br /> ar << [:dqo, index, 1] <br /> open_quotes += 1 <br /> <br /> elsif m =~ /\A\S"\z/ <br /> <br /> ar << [:dqc, index, 1] <br /> close_quotes += 1 <br /> <br /> <br /> else <br /> <br /> <br /> if m =~ /\A\s"\s\z/ # " surrounded by whitespace <br /> <br /> if open_quotes > close_quotes <br /> <br /> ar << [:dqc, index, 1] <br /> close_quotes += 1 <br /> <br /> # avoid :sqo followed by :dqc or :sqc followed by :dqc <br /> if post_match =~ /"/ && open_quotes == close_quotes && (ar.at(-2).first == :sqo || ar.at(-2).first == :sqc) <br /> ar.pop <br /> ar << [:dqo, index, 1] <br /> close_quotes -= 1 <br /> open_quotes += 1 <br /> end <br /> <br /> else <br /> <br /> ar << [:dqo, index, 1] <br /> open_quotes += 1 <br /> <br /> end <br /> <br /> <br /> elsif m =~ /\A\s'\s\z/ # ' surrounded by whitespace <br /> <br /> if open_quotes > close_quotes <br /> <br /> ar << [:sqc, index, 1] <br /> close_quotes += 1 <br /> <br /> # avoid :dqo followed by :sqc or :dqc followed by :sqc <br /> if post_match =~ /'/ && open_quotes == close_quotes && (ar.at(-2).first == :dqo || ar.at(-2).first == :dqc) <br /> ar.pop <br /> ar << [:sqo, index, 1] <br /> close_quotes -= 1 <br /> open_quotes += 1 <br /> end <br /> <br /> else <br /> <br /> ar << [:sqo, index, 1] <br /> open_quotes += 1 <br /> <br /> end <br /> <br /> <br /> elsif m =~ /\A\s["']["']+\z/ # :qoblock: xxx "'""'abc yyy <br /> <br /> l = m[1..-1].strip.length <br /> ar << [:qoblock, index, l] <br /> old_open_quotes = open_quotes <br /> open_quotes += l <br /> <br /> if close_quotes == 0 && old_open_quotes == 0 && open_quotes % 2 == 0 && post_match !~ /["']/ <br /> ret << m[2..-2] <br /> open_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> <br /> elsif m =~ /\A[^"'\s]["']["']+\z/ # :qcblock: xxx abc"'""' yyy <br /> <br /> l = m[1..-1].strip.length <br /> ar << [:qcblock, index+l-1, l] <br /> old_close_quotes = close_quotes <br /> close_quotes += l <br /> <br /> if open_quotes == 0 && old_close_quotes == 0 && close_quotes % 2 == 0 && post_match !~ /["']/ <br /> ret << m[2..-2] <br /> close_quotes = 0 <br /> ar.pop <br /> next <br /> end <br /> <br /> <br /> elsif m =~ /\A\s*["'].*?["']\s*\z/ # last try (experimental) <br /> <br /> ret.concat(m.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? }) <br /> next <br /> <br /> <br /> elsif m =~ /\A[^"']+[^"'\s]\z/ # part of quoted substring contains neither " nor ' <br /> next unless open_quotes == 0 && close_quotes == 0 <br /> next if m.strip.empty? <br /> ret << m.gsub(/\x20\x20/, "\x20").strip; next <br /> end <br /> <br /> end <br /> <br /> puts <br /> puts "open_quotes: #{open_quotes}\nclose_quotes: #{close_quotes}\n" <br /> #puts "ar: #{ar.inspect}" <br /> <br /> if open_quotes == close_quotes <br /> <br /> #puts "open_quotes & close_quotes: #{close_quotes}" <br /> puts "ar: #{ar.inspect}" <br /> <br /> ret << s[ar.first[1]..ar.last[1]].gsub(/\x20\x20/, "\x20")[1..-2] unless ar.empty? <br /> <br /> ar.clear <br /> open_quotes = 0 <br /> close_quotes = 0 <br /> <br /> end <br /> <br /> end # scan 2 <br /> <br /> unless open_quotes.zero? && close_quotes.zero? <br /> error_code = 1 <br /> puts "\e[1mparsing error\e[m for the quoted string: #{str.strip.squeeze[0..20]}" <br /> #raise "\e[1mparsing error\e[m for the quoted string: #{str.strip.squeeze[0..20]}" <br /> end <br /> <br /> end # if <br /> <br />end # scan 1 <br /> <br /> <br /> <br />num_of_chars2 = ret.join.count('a-zA-Z_0-9', "^\000ds") <br /> <br />unless num_of_chars1 == num_of_chars2 <br /> error_code = 1 <br /> puts "\n\e[1mparsing error due to wrong number of characters a-zA-Z_0-9\e[m: \n#{num_of_chars2} instead of #{num_of_chars1}\n" <br /> #raise "\e[1mparsing error due to wrong number of characters a-zA-Z_0-9\e[m: \n#{num_of_chars2} instead of #{num_of_chars1}\n in #{str.strip.squeeze[0..20]}" <br />end <br /> <br /> <br /># use Shellwords in case the quote matching above failed <br />if error_code == 1 <br />#if error_code == 1 || ret.join =~ /\A["']+\z/ <br /> require 'shellwords' <br /> ret.clear <br /> ret.concat(Shellwords::shellwords(str)) <br /> #str =~ /\A\S+\z/ ? ret.concat(str.split(/"(.+?)"|\s+/).reject {|sm| sm.empty? }) : ret.concat(Shellwords::shellwords(str)) <br />end <br /> <br /> <br /> <br />puts "\n\e[1mResult\e[m:\n\n" <br />ret.each_with_index do |t,i| <br /> # decode encoded escaped quotes <br /> t = t.gsub(/\000d\000|\000s\000/) { |m| m =~ /^\000d\000$/ ? '\"' : "\\'" } <br /> puts "#{i+1}: #{t.inspect}" <br />end <br /> <br />puts "\n\e[1mShellwords\e[m:\n\n" <br />require 'shellwords' <br />Shellwords::shellwords(str2).each_with_index { |t,i| puts "#{i+1}: #{t.inspect}" } <br /> <br /> <br />#---------------------- <br /> <br /> <br /># matching quoted strings using backreferences <br /># See: Regexes in Depth: Advanced Quoted String Matching, <br /># http://blog.stevenlevithan.com/archives/match-quoted-string <br /> <br />str = '"abc"' <br /> <br />regex = %r{(["'])([^"']*)(\1)} <br />regex = %r{(["'])([^\1]*)(\1)} <br />p regex <br /> <br />str.scan(regex) { |m| p m; p $1 << $2 << $3 } <br /></code> Sun, 04 May 2008 18:47:17 GMT http://snippets.dzone.com/posts/show/5459 ntk ()