DZone Snippets: scrape code

Converting XHTML to XML

jrobertson (James Robertson) — Sun, 10 Feb 2008 15:35:40 GMT

Based on the code from 'Convert from HTML to XML with HTML Tidy', this code will read an xhtml file and extract text to gallery.xml as instructed by xhtml2xml.xml



#!/usr/bin/ruby

  

  require 'tidy'

  require 'projxslt'

  

  FILE_PATH = "../"

  

  class Xhtml2Xml

    def convert()

      project = 'xhtml2xml'

      filein = 'xhtml2xml.xml'

      filehtml = 'gallery.html'

      filexml = 'gallery_xhtml.xml'

      xslfile_temp = 'gallery.xsl'

      xslfile = 'xhtml2xml.xsl'

      fileout = 'gallery.xml'

      tidy_config = 'tidy.txt'

      

      project_path = FILE_PATH + project + '/'

      tidy_config_path = project_path + tidy_config

      filein_path = project_path + filein

      filehtml_path = project_path + filehtml

      filexml_path = project_path + filexml

      xslfile_temp_path = project_path + xslfile_temp

      xslfile_path = project_path + xslfile

      fileout_path = project_path + fileout

      

      Tidy.path = '/usr/lib/libtidy.so'



      file = File.new(filehtml_path,'r')

      buffer = file.read

      xml = Tidy.open(:show_warnings=>true) do |tidy|

        tidy.options.output_xml = true

        tidy.load_config(tidy_config_path)

        puts tidy.options.show_warnings

        xml = tidy.clean(buffer)

        puts tidy.errors

        puts tidy.diagnostics

        xml

      end

      

      #strip out the html document type declaration and save the file

      html_declaration = xml[/]*>){2}/]

      save_file(filexml_path, xml.gsub(html_declaration,''))    

      transform(filein_path, xslfile_path, xslfile_temp_path)

      transform(filexml_path, xslfile_temp_path, fileout_path)

      

    end

    

    def transform(xml_filepath, xsl_filepath, save_filepath)

      pxsl = Projxslt.new(xml_filepath, xsl_filepath)

      outfile = pxsl.transform

      save_file(save_filepath, outfile)

    end

    

    def save_file(filepath, buffer)

      file = File.new(filepath,'w') 

      file.puts buffer

      file.close

    end    

  end

  

  if __FILE__ == $0

    h2x = Xhtml2Xml.new()

    h2x.convert()

  end

file: xhtml2xml.xml

file:xhtml2xml.xsl (transforms the file xhtml2xml.xml to file gallery.xsl)







  

    :

    

    

      

        http://www.w3.org/1999/XSL/Transform

      

      

        1.0

      

      





  

    xml

  

  

    yes

  











      

        html

      



      

      



      

        

          

        

                  



  

    

      

        

      

      

        

      

    

    

  



    



        

       

              

        

            

              normalize-space($

                

                )

output: gallery.xml (this file is the product of gallery_xhtml.xml and gallery.xsl)







  Journey to Windsor

  

    Windsor Castle

    July 2003

    dscn0824.jpg

    

      A bright, red mailbox inside the castle. It seems oddly familiar in an historic setting.

Scrape an XHTML document using Ruby

jrobertson (James Robertson) — Sun, 27 Jan 2008 14:09:24 GMT

A simple Ruby script to scrape an XHTML file with the selected content being saved to an xml file ready for transformation into an RSS feed. This example uses the XHTML file from http://newsgang.net/audio/ which is then saved locally as 'thegang.xml'.



#!/usr/bin/ruby

# file: thegang.rb



require 'rexml/document'

include REXML



class TheGang

  def initialize()

  end

  

  def rssify()

    file = File.new('thegang.xml','r')

    doc = Document.new(file)

    rss_doc = Document.new

    root = Element.new('rss')

    rss_doc.add_element(root)

    

    doc.root.elements.each("body/div/ul/li/h2/a") do |node|    

      o_rssitem = Element.new('item')

      o_li = node.parent.parent

      

      o_rsstitle = Element.new('title')

      o_rsstitle.text = node.text.gsub(/[\n,' ']/,'')

      o_rssitem.add_element(o_rsstitle)

      

      o_rsshref_audio = Element.new('href_audio')

      o_rsshref_audio.text = node.attributes.get_attribute('href').to_s.gsub('amp;&','')      

      o_rssitem.add_element(o_rsshref_audio)

      

      o_rsshref = Element.new('href')

      o_rsshref.text = o_rsshref_audio.text.gsub('&from=audio','')      

      o_rssitem.add_element(o_rsshref)

      

      o_rssdate = Element.new('date')

      o_rssdate.text = "#{o_li.elements["p/span[1]"].text} #{o_li.elements["p/span[2]"].text}"

      o_rssitem.add_element(o_rssdate)

      rss_doc.root.add_element(o_rssitem)

      

    end



    file = File.new('thegang_rss.xml','w')

    file.puts rss_doc

    file.close

  end

end





if __FILE__ == $0

  gang = TheGang.new

  gang.rssify

end

see also: www.dapper.net

output (extract)





  TheGangXII-II/gangitem/id=6501&from=audio/gangitem/id=6501Jan 25

  TheGangXII-I/gangitem/id=6499&from=audio/gangitem/id=6499Jan 25

  NewsGangLive01.24.08/gangitem/id=6445&from=audio/gangitem/id=6445Jan 24

  NewsGangLiveII/gangitem/id=6377&from=audio/gangitem/id=6377Jan 23

  ...

Screen scape heise.de newsticker (german)

tsch () — Sun, 13 Nov 2005 03:04:25 GMT



#!/usr/bin/env python

# -*- encoding: latin1 -*-



import BeautifulSoup

from PyRSS2Gen import RSSItem, Guid

import ScrapeNFeed

import urllib2

import re



debug = 0



def fetch(url):

    response = urllib2.urlopen(urllib2.Request(url))

    return response.read(),response.info()



class HeiFeed(ScrapeNFeed.ScrapedFeed):    

    def HTML2RSS(self, headers, body):

        items = []

        soup = BeautifulSoup.BeautifulSoup(body)

        for item in soup('a', {'href' : re.compile('^meldung.*')}):

            link = 'http://www.heise.de/newsticker/' + item['href']

            if not self.hasSeen(link):

                title = item.contents[0].strip()

                if debug:

                    print "title: " + title

                    print "link : " + link

                response, headers = fetch(link)

                s = BeautifulSoup.BeautifulSoup(response)

                desc = s.fetch('div',{'class':'meldung_wrapper'})[0].prettify()

                items.append(RSSItem(title=title, description=desc, link=link))

            self.addRSSItems(items)



HeiFeed.load("heise.de newsticker", 'http://www.heise.de/newsticker/',

             "heise.de newsticker", 'heise_rss.xml', 'heise_rss.pickle',

             managingEditor = 'tsch')