Difference between revisions of "Wordman/ProjectGezlak"

Revision as of 06:09, 8 June 2010

• Back to Wordman/CommissionEngine

I'm starting Project Gezlak (named after a type of metal elemental that roams Autochthonia) to try to automatically fix the links that were broken during the process of converting to MediaWiki. This would probably be a lot easier with direct access to the server, but I don't have that.

The idea is to use the Python wikipedia robot framework to examine page content, detect certain kinds of damaged links, and repair them.

The script has been tested on a handful of pages, and seems to work; however, I can't rule out all mistakes. Tell me about these in the following sections:

Errors

If you notice the script had errors, list them here.

• error

Opt-out

If you'd rather keep a certain page out of this automatic cleanup, list it here:

• page

Code

#!/usr/bin/python

import wikipedia as pywikibot
import pagegenerators
import re
import time

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
   '&params;': pagegenerators.parameterHelp
}

class AllPagesBot:
   def __init__(self, generator, dry):
      """
      Constructor. Parameters:
         * generator - The page generator that determines on which pages
                    to work on.
         * dry      - If True, doesn't do any real changes, but only shows
                    what would have been changed.
      """
      #self.generator = pagegenerators.AllpagesPageGenerator()
      self.generator = pagegenerators.TextfilePageGenerator('exaltedall.txt')
      self.dry = dry
      self.pages = {}
      
   def run(self):
      while True:
         try:
           page = self.generator.next()
           if page is None:
             break;
           aslink = page.aslink()
           aslink = aslink.replace('[[','')
           aslink = aslink.replace(']]','')
           self.pages[aslink] = page
         except:
           break
      #for link in self.pages.keys():
      #   print "\"%s\"," % (link)
         
class ExaltedRepairBot:
   # Edit summary message that should be used.
   # NOTE: Put a good description here, and add translations, if possible!
   msg = {
      'en': u'Testing script to fix links messed up in conversion'
   }

   def __init__(self, generator, dry, quiet, allpages):
      """
      Constructor. Parameters:
         * generator - The page generator that determines on which pages
                    to work on.
         * dry      - If True, doesn't do any real changes, but only shows
                    what would have been changed.
      """
      self.allpages = allpages
      self.generator = generator
      self.dry = dry
      self.quiet = quiet
      # Set the edit summary message
      self.summary = pywikibot.translate(pywikibot.getSite(), self.msg)
      self.linkPat = re.compile(r'\[\[(.+?)\]\]')
      self.splitPat = re.compile(r'\s*\|\s*')
      self.levelPat = re.compile(r'\s*/\s*')

   def run(self):
      for page in self.generator:
         try:
            self.treat(page)
         except Exception, err:
            print "   ERROR: Failed to process page %s when examining link %s: %s\n     %s" % (page.aslink(), self.curLink, err, self.curLine)
         
   def treat(self, page):
      """
      Loads the given page, does some changes, and saves it.
      """
      print ""
      print "Processing page %s..." % (page.aslink())
      try:
         # Load the page
         text = page.get()
      except pywikibot.NoPage:
         pywikibot.output(u"Page %s does not exist; skipping."
                      % page.aslink())
         return
      except pywikibot.IsRedirectPage:
         pywikibot.output(u"Page %s is a redirect; skipping."
                      % page.aslink())
         return

      ################################################################
      # NOTE: Here you can modify the text in whatever way you want. #
      ################################################################

      # If you find out that you do not want to edit this page, just return.
      # Example: This puts the text 'Test' at the beginning of the page.
      lines = text.splitlines(1)
      newlines = [];
      for line in lines:
         newline = self.treatLine(line)
         newlines.append(newline)
      text = "".join(newlines)

      # only save if something was changed
      if text != page.get():
         if not self.quiet:
            # Show the title of the page we're working on.
            # Highlight the title in purple.
            pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
                         % page.title())
            # show what was changed
            pywikibot.showDiff(page.get(), text)
         if not self.dry:
            if self.quiet:
               choice = 'y'
            else:
               choice = pywikibot.inputChoice(u'Do you want to accept these changes?', ['Yes', 'No'], ['y', 'N'], 'N')
            if choice == 'y':
               try:
                  # Save the page
                  page.put(text, comment=self.summary)
               except pywikibot.LockedPage:
                  pywikibot.output(u"Page %s is locked; skipping."
                               % page.aslink())
               except pywikibot.EditConflict:
                  pywikibot.output(
                     u'Skipping %s because of edit conflict'
                     % (page.title()))
               except pywikibot.SpamfilterError, error:
                  pywikibot.output(u'Cannot change %s because of spam blacklist entry %s'
                     % (page.title(), error.url))
      else:
         print "   ...no changes."
         time.sleep(3) # Don't flood the server

   def treatLine(self, line):
      str_list = []
      lastPos = 0
      matches = self.linkPat.finditer(line)
      for match in matches:
         fulllink = match.group(0)
         link = match.group(1)
         end = match.end(1)
         try:
            (newLink, end) = self.treatLink(line, link, end)
         except Exception, err:
            print "   ERROR: Failed to process link %s: %s\n     %s\n     Continuing leaving link unaltered" % (self.curLink, err, self.curLine)
            newLink = link
         start = match.start(1)
         str_list.append(line[lastPos:start])
         str_list.append(newLink)
         lastPos = end
         if not self.quiet and link != newLink:
            print "   %s -> %s" % (link,newLink)
      str_list.append(line[lastPos:])
      return ''.join(str_list)

   def treatLink(self, line, link, end):
      self.curLink = link
      self.curLine = line
      sourceLast = end
      theSplit = self.splitPat.split(link)
      link = theSplit[0]
      label = None
      if len(theSplit) > 1:
         label = theSplit[1]
      if not self.pageExists(link):
         (link,label,sourceLast) = self.repairMissingLink(line, link, label, sourceLast)
      #if self.pageExists(link):
      #   (link,label,sourceLast) = self.repairExistingLink(line, link, label, sourceLast)
      else:
         print "   Warning: after repairs, link %s still not found. Using anyway." % (link)
      treated = link
      if label:
         treated += '|' + label
      return (treated,sourceLast)

   def pageExists(self, link):
      cleanLink = link.replace('_',' ')
      cleanLink = cleanLink[0].capitalize() + cleanLink[1:]
      return self.allpages.has_key(cleanLink)

   def repairExistingLink(self, line, link, label, end):
      # "Prettify links starting with 'wordman'
      if link.startswith('Wordman') and not label:
         levels = self.levelPat.split(link)
         if len(levels) > 1:
            mainpage = levels[0]
            subpage = levels[1]
            name = self.space_out_camel_case(subpage).strip()
            label = name
      return (link, label, end)

   def repairMissingLink(self, line, link, label, end):
      if link.startswith('['):
         (link, label, end) = self.repairBadLabeledLink(line, link, label, end)
      levels = self.levelPat.split(link)
      if len(levels) > 2:
         (link, label, end) = self.repairMultiLevelLink(line, link, label, end, levels)
      return (link, label, end)

   def repairBadLabeledLink(self, line, link, label, end):
      # First, get rid of the leading brackets in the link
      link = link.replace('[','')
      # Becuase of the pattern matching that got us this far, we know that the characters
      # at line[end] are two closing brackets. Skip past them, then get everything until
      # the next two closing brackets, which should be an attempt at a label that got corrupted.
      assumption = line[end:end+2]
      if assumption != ']]':
         raise AssertionError 
      end += 2
      remainder = line[end:]
      index = remainder.find(']]')
      if index == -1:
         raise AssertionError
      remainder = remainder[:index]
      if index < 2:
         # In this case, it looks like there might be a link like [[[[link]]]] or
         # [[[link]] ]], so eliminate the extra brackets
         pass
      elif remainder.startswith('\''):
         # In this case, there is a link with an apostrophe that got scrambled,
         # like [[[page/sub]]'page]]. Append the remainder to the link.
         link += remainder
         end += index
      else:
         # Probably a label, like [[[link]] | label]]
         (nothing,newlabel) = self.splitPat.split(remainder)
         label = newlabel.strip()
         # Tell the caller to ignore the extra line source we just consumed
         end += index
      return (link, label, end)

   def repairMultiLevelLink(self, line, link, label, end, levels):
      # In this case, the problem is usually that the name of and a/b page was just
      # prepended blindly, rather than just the a. So, build a link from the first
      # and last level.
      testlink = levels[0] + '/' + levels[-1]
      # If this link exists, use it
      link = testlink;
      return (link, label, end)

   def space_out_camel_case(self, str, delimiter=' '):
      """Adds spaces to a camel case string.  Failure to space out string returns the original string.
      >>> space_out_camel_case('DMLSServicesOtherBSTextLLC')
      'DMLS Services Other BS Text LLC'
      """
      text = re.sub('((?=[A-Z][a-z])|(?<=[a-z])(?=[A-Z]))', delimiter, str).strip()
      if text != str:
         # Uncapitalize articles
         text = text.replace(' Of ', ' of ')
         text = text.replace(' In ', ' in ')
         text = text.replace(' To ', ' to ')
         text = text.replace(' At ', ' at ')
         text = text.replace(' The ', ' the ')
         text = text.replace(' And ', ' and ')
      return text

def main():
   # This factory is responsible for processing command line arguments
   # that are also used by other scripts and that determine on which pages
   # to work on.
   genFactory = pagegenerators.GeneratorFactory()
   # The generator gives the pages that should be worked upon.
   gen = None
   # This temporary array is used to read the page title if one single
   # page to work on is specified by the arguments.
   pageTitleParts = []
   # If dry is True, doesn't do any real changes, but only show
   # what would have been changed.
   dry = False
   quiet = False

   # Parse command line arguments
   for arg in pywikibot.handleArgs():
      if arg.startswith("-dry"):
         dry = True
      elif arg.startswith("-quiet"):
         quiet = True
      else:
         # check if a standard argument like
         # -start:XYZ or -ref:Asdf was given.
         if not genFactory.handleArg(arg):
            pageTitleParts.append(arg)

   if pageTitleParts != []:
      # We will only work on a single page.
      pageTitle = ' '.join(pageTitleParts)
      page = pywikibot.Page(pywikibot.getSite(), pageTitle)
      gen = iter([page])

   if not gen:
      gen = genFactory.getCombinedGenerator()
   if gen:
      # The preloading generator is responsible for downloading multiple
      # pages from the wiki simultaneously.
      gen = pagegenerators.PreloadingGenerator(gen)
      allbot = AllPagesBot(gen, dry)
      allbot.run()
      bot = ExaltedRepairBot(gen, dry, quiet, allbot.pages)
      bot.run()
   else:
      pywikibot.showHelp()

if __name__ == "__main__":
   try:
      main()
   finally:
      pywikibot.stopme()

Comments

Great! I look forward to seeing the results. Thankyou! :) nikink