User:Image optimisation bot/Source/ImageOptimisationBot.java

package com.wikia.runescape;

import java.io.*; import java.net.URLEncoder; import java.util.*; import java.util.logging.*; import java.util.regex.*;

import javax.security.auth.login.*;

import org.wikipedia.Wiki;

/** * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 3 of the License, or (at your option) any later * version. This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. You should have received a copy of the GNU General Public License * along with this program. If not, see &lt;http://www.gnu.org/licenses/&gt;. */ public class PNGOptimisationBot { private static final Logger log = Logger.getLogger("com.wikia.runescape");

static { log.setLevel(Level.INFO); }

private static final Pattern requestRegex = Pattern.compile("^\\*([^:|]+:[^:|]+)\\s*$");

/**  * @param args *           unused */ public static void main(String[] args) { // Read the bot's configuration file. Settings settings = new Settings(new File(System.getProperty("user.home"), ".pngoptbot.conf")); // Require some things out of it from the start... {     boolean fatalError = false; if (settings.getProperty("Wiki") == null) { log.log(Level.SEVERE, "$HOME/.pngoptbot.conf does not contain a value for Wiki, the wiki to work on"); fatalError = true; }     if (settings.getProperty("LoginName") == null) { log.log(Level.SEVERE, "$HOME/.pngoptbot.conf does not contain a value for LoginName, the username of the bot account on the wiki"); fatalError = true; }     if (settings.getProperty("LoginPassword") == null) { log.log(Level.SEVERE, "$HOME/.pngoptbot.conf does not contain a value for LoginPassword, the password of the bot account on the wiki"); fatalError = true; }     if (settings.getProperty("RequestPage") == null) { log.log(Level.SEVERE, "$HOME/.pngoptbot.conf does not contain a value for RequestPage, the name of the bot's request page on the wiki"); fatalError = true; }     if (settings.getProperty("MinimumCompressionRatio") == null) { log.log(Level.SEVERE, "$HOME/.pngoptbot.conf does not contain a value for MinimumCompressionRatio, the value (between 1 and 99) determining the minimum compression ratio required to reupload an optimised file to the wiki"); fatalError = true; }     if (fatalError) { System.exit(1); return; }   }

// Which wiki are we working on? Wiki wiki = new Wiki(settings.getProperty("Wiki"), settings.getProperty("ScriptPath", "")); wiki.setUsingCompressedRequests(false); // WIKIA, Y U NO GZIP?

loginLoop: while (true) { // LOGIN LOST LOOP while (true) // Retry loop for network errors try { wiki.login(settings.getProperty("LoginName"), settings.getProperty("LoginPassword").toCharArray); break; } catch (FailedLoginException e) { log.log(Level.SEVERE, "Login failed; please check LoginName and LoginPassword in $HOME/.pngoptbot.conf", e); System.exit(1); return; } catch (IOException e) { log.log(Level.INFO, "Network error occurred while logging in; retrying shortly", e); shortDelay; }

// Process the bot's request page, starting from the revision after // the last one seen, or null if the bot has not seen a request yet. Long lastRequestRevision = null; if (settings.getProperty("LastRequestRevision") != null) { lastRequestRevision = Long.parseLong(settings.getProperty("LastRequestRevision")); }

/* MAIN BOT LOOP */ while (true) { Wiki.Revision[] revisions = new Wiki.Revision[0]; while (true) // Retry loop for network errors try { revisions = wiki.getPageHistory(settings.getProperty("RequestPage"), null, lastRequestRevision != null ? lastRequestRevision + 1 : null); break; } catch (IOException e) { log.log(Level.INFO, "Network error occurred while getting revisions; retrying shortly", e); shortDelay; }

// Sort the revisions by ID, i.e. from oldest at [0] to newest // at [length - 1]. Arrays.sort(revisions, new Comparator&lt;Wiki.Revision&gt; {         public int compare(Wiki.Revision o1, Wiki.Revision o2) {            return Long.signum(o1.getRevid - o2.getRevid);          }        });

/*        * If a week has passed since the last optimisation of the * entire wiki's PNG images, mark all revisions as done (but        * really skipped) and optimise the entire wiki instead. */

if (new Date.getTime &gt;= Long.parseLong(settings.getProperty("LastEntireWikiOptimisation", Long.toString(Long.MIN_VALUE))) + 7L * 86400 * 1000) { // ENTIRE WIKI (USE SPECIAL:ALLPAGES) log.log(Level.INFO, "Starting optimisation of PNG images on the entire wiki");

// Get Special:AllPages for the File namespace. String[] allFiles = new String[0]; while (true) try { allFiles = wiki.listPages("", Wiki.NO_PROTECTION, Wiki.FILE_NAMESPACE, 0, Integer.MAX_VALUE); break; } catch (IOException e) { log.log(Level.WARNING, "Network error occurred while getting Special:AllPages"); }

// Optimise ALL the images! for (String file : allFiles) { try { optimize(wiki, settings, file, "(Automated) PNG recompression"); } catch (LoginException e) { continue loginLoop; } catch (IOException e) { log.log(Level.INFO, "Network error occurred while optimising an image; trying another from Special:AllPages shortly", e); shortDelay; }         }

// Mark the current timestamp as being the last run for the // entire wiki. settings.setProperty("LastEntireWikiOptimisation", Long.toString(new Date.getTime)); if (revisions.length &gt; 0) settings.setProperty("LastRequestRevision", Long.toString(revisions[revisions.length - 1].getRevid)); try { settings.store; } catch (IOException e) { log.log(Level.WARNING, "Cannot write LastEntireWikiOptimisation to settings; the images on the entire wiki may be retried on the next bot run", e); }

log.log(Level.INFO, "Optimisation of PNG images on the entire wiki done"); } else { // REQUESTS ON PAGE for (Wiki.Revision revision : revisions) { lastRequestRevision = revision.getRevid; settings.setProperty("LastRequestRevision", lastRequestRevision.toString); try { settings.store; } catch (IOException e) { log.log(Level.WARNING, "Cannot write LastRequestRevision to settings; some images may be retried on the next bot run", e); }           String requestor = revision.getUser; if (requestor != null /*- i.e. not revision deleted */) { String text = null; while (true) // Retry loop for network errors try { text = revision.getText; break; } catch (IOException e) { log.log(Level.INFO, "Network error occurred while getting the request page's text for a revision; retrying shortly", e); shortDelay; }

// Parse requests out of the revision's new text. String[] lines = text.replace("\r\n", "\n").split("\n"); Set&lt;String&gt; imageNames = new HashSet&lt;String&gt;; for (String line : lines) { // To be considered requests, lines must start // with * and contain 2 :-separated tokens. // e.g. *File:Blah.png Matcher m = requestRegex.matcher(line); if (m.find) { imageNames.add(m.group(1)); }             }

// Do not bother to arrange continuing this request // in case it's interrupted. In PNGOptimisationBot, // requests are only a HINT. for (String imageName : imageNames) { while (true) try { optimize(wiki, settings, imageName, String.format("(Semi-automated) PNG recompression requested by %1$s (talk | contribs)", revision.getUser)); break; } catch (LoginException e) { continue loginLoop; } catch (IOException e) { log.log(Level.INFO, "Network error occurred while optimising an image; trying another request shortly", e); shortDelay; }             }            }          }        }

// Whether all revisions are done or the entire wiki was just // optimised, wait for the next request-page check. try { try { Thread.sleep(Long.parseLong(settings.getProperty("RunInterval", "3600")) * 1000); } catch (NumberFormatException e) { log.log(Level.WARNING, "Incorrect run interval; please check RunInterval in $HOME/.pngoptbot.conf (using 3600 seconds)"); Thread.sleep(3600 * 1000); }       } catch (InterruptedException e) { // don't care }     }    }  }

/**  * Settings are Properties that automatically load and store themselves into * files. Reads and writes ignore &lt;tt&gt;IOException&lt;/tt&gt;s; the errors are * logged to Java Logging instead. */ private static class Settings extends Properties { private static final long serialVersionUID = 1L;

private final File file;

public Settings(File file) { this.file = file; try { InputStream in = new FileInputStream(file); try { load(in); } finally { in.close; }     } catch (IOException e) { log.log(Level.WARNING, "Settings file cannot be read; using no settings at all", e); }   }

public void store throws IOException { OutputStream out = new FileOutputStream(file); try { store(out, null); } finally { out.close; }   }  }

/**  * A cache of the last revisions of images seen by the bot. * &lt;p&gt; * The cache is stored in ${user.home}/.cache/pngoptbot using the canonical * name of each file URL-encoded. This avoids path traversal vulnerabilities * and allows certain characters to be used safely in filesystems * prohibiting them. */ private static class RevisionCache { public static Calendar get(String fileNamespaceAndName) { File cacheFile = new File(new File(new File(System.getProperty("user.home"), ".cache"), "pngoptbot"), canonicalName(fileNamespaceAndName)); try { ObjectInputStream ois = new ObjectInputStream(new FileInputStream(cacheFile)); try { return (Calendar) ois.readObject; } catch (ClassNotFoundException e) { return null; } finally { ois.close; }     } catch (IOException e) { return null; }   }

public static void set(String fileNamespaceAndName, Calendar timestamp) { File cacheDirectory = new File(new File(System.getProperty("user.home"), ".cache"), "pngoptbot"); cacheDirectory.mkdirs; File cacheFile = new File(cacheDirectory, canonicalName(fileNamespaceAndName)); try { ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(cacheFile)); try { oos.writeObject(timestamp); } finally { oos.close; }     } catch (IOException e) { log.log(Level.WARNING, "Cannot write revision number to cache for " + fileNamespaceAndName); }   }  }

private static void shortDelay { try { Thread.sleep(45000); } catch (InterruptedException e) { // don't care } }

/**  * Reads, attempts to losslessly recompress, and uploads back to the wiki if   * the recompressed image is much smaller than the original, an image on a   * certain wiki. *   * @param wiki *           The wiki to work on. * @param settings *           The settings to use for the optimisation. * @param fileNamespaceAndName *           The name of the page, including its namespace, to optimise. * @param editReason *           The edit reason to use for the upload. * @throws LoginException *            if the login becomes incorrect while reading from, or writing *            to, the wiki * @throws IOException *            if a retriable error occurs while reading from, or writing *            to, the wiki */ private static void optimize(Wiki wiki, Settings settings, String fileNamespaceAndName, String editReason) throws LoginException, IOException { // What timestamp is this image at on the wiki? // (Note: The API doesn't like having File:X Y Z.png, so replace with _) Wiki.LogEntry[] logEntries = wiki.getImageHistory(fileNamespaceAndName.replaceAll("^[^:]+:", "").replace(' ', '_')); // Sort the log entries by timestamp, earliest to latest. Arrays.sort(logEntries, new Comparator&lt;Wiki.LogEntry&gt; {     public int compare(Wiki.LogEntry o1, Wiki.LogEntry o2) {        return o1.getTimestamp.compareTo(o2.getTimestamp);      }    });

Calendar wikiTimestamp = logEntries[logEntries.length - 1].getTimestamp; // In the following line we avoid uploading a file if we are the user // who last uploaded it. This gets rid of retries for // obviously-optimised images. if (wikiTimestamp != null &amp;&amp; logEntries[logEntries.length - 1].getTarget != null /*- i.e. not revision deleted */&amp;&amp; logEntries[logEntries.length - 1].getUser != null &amp;&amp; !logEntries[logEntries.length - 1].getUser.getUsername.equals(settings.getProperty("LoginName"))) { // What timestamp do we have on file? Calendar seenTimestamp = RevisionCache.get(fileNamespaceAndName); if (seenTimestamp == null || wikiTimestamp.compareTo(seenTimestamp) &gt; 0 /*- i.e. the wiki timestamps compares later than what we have on file */) { File localFile; FileOutputStream localFileOut; // Newer, or never seen? Try optimising the file. // Create a local file to hold the contents first. try { localFile = File.createTempFile("pngoptbot-", ".png"); localFileOut = new FileOutputStream(localFile); } catch (IOException e) { log.log(Level.SEVERE, "Cannot create a temporary file to hold the contents of " + fileNamespaceAndName + " before optimisation; optimisation is cancelled for the file"); return; }       // Read contents from the wiki. try { byte[] imageData = wiki.getImage(fileNamespaceAndName.replaceAll("^[^:]+:", "")); int oldLength = imageData.length; // Is it a PNG image? /*          * A PNG file starts with an 8-byte signature. The * hexadecimal byte values are 89 50 4E 47 0D 0A 1A 0A; * [...] ~Wikipedia */         if (oldLength &gt;= 8 &amp;&amp; imageData[0] == (byte) 0x89 &amp;&amp; imageData[1] == (byte) 0x50 &amp;&amp; imageData[2] == (byte) 0x4E &amp;&amp; imageData[3] == (byte) 0x47 &amp;&amp; imageData[4] == (byte) 0x0D &amp;&amp; imageData[5] == (byte) 0x0A &amp;&amp; imageData[6] == (byte) 0x1A &amp;&amp; imageData[7] == (byte) 0x0A) { try { // And write to the local file. localFileOut.write(imageData); imageData = null; } catch (IOException e) { log.log(Level.SEVERE, "Cannot write to a temporary file to hold the contents of " + fileNamespaceAndName + " before optimisation; optimisation is cancelled for the file"); return; }           // Now, with the contents of the file stored on the // local filesystem, some tools may be launched on it. String[][] toolCommandLines = { { "optipng", "-zc8-9", "-zm8-9", "-zs0-3", "-f0-5", localFile.toString }, { "advpng", "-z4", localFile.toString }, { "advdef", "-z4", localFile.toString } }; for (String[] toolCommandLine : toolCommandLines) { try { Process p = Runtime.getRuntime.exec(toolCommandLine); p.waitFor; } catch (IOException e) { log.log(Level.SEVERE, "Cannot launch " + toolCommandLine[0] + "! Make sure it is installed and appears in your PATH environment variable!"); System.exit(1); } catch (InterruptedException e) { log.log(Level.WARNING, "Received java.lang.InterruptedException while executing " + toolCommandLine[0] + "; optimisation is cancelled for the file"); return; }           }            // Now look at the size of the file, and if it's            // compressed by X% or more of the original size, // reupload it to the wiki. long newLength = localFile.length; if (newLength &gt;= 8 /*- must have PNG header */&amp;&amp; (1.0 - (double) newLength / (double) oldLength) * 100.0 &gt;= Double.parseDouble(settings.getProperty("MinimumCompressionRatio"))) { wiki.upload(localFile, fileNamespaceAndName.replaceAll("^[^:]+:", ""), "", editReason); }         }          RevisionCache.set(fileNamespaceAndName, wikiTimestamp); } finally { localFile.delete; }     }    }  }

protected static String canonicalName(String name) { try { StringBuilder nameBuffer = new StringBuilder(name); if (nameBuffer.length &gt; 0) { // Uppercase the first letter of the file name. nameBuffer.setCharAt(0, Character.toUpperCase(nameBuffer.charAt(0))); // If this is a namespaced name, the namespace's first letter // was uppercased above instead. Now uppercase the first letter // of the file name. int colonIndex = nameBuffer.indexOf(":"); if (colonIndex != -1 &amp;&amp; nameBuffer.length &gt; colonIndex + 1) { nameBuffer.setCharAt(colonIndex + 1, Character.toUpperCase(nameBuffer.charAt(colonIndex + 1))); }     }      return URLEncoder.encode(nameBuffer.toString.replace(' ', '_'), "UTF-8"); } catch (UnsupportedEncodingException shouldNeverHappen) { throw new InternalError("UTF-8 is not supported by the Java VM"); } } }