Mediawiki LocalServer

Mediawiki offline support thanks to Google Gears

This is an attempt to provide an offline version of the wiki using Google Gears LocalServer.
Inspired by this blog post

To be able to use Mediawiki offline, the Mediawiki website has to have this extension installed and the browser has to have the Google Gears extension.

The original blog post was quick and dirty (do it in less than one hour), but too dirty for me:

• There was always a complete replication of the full wiki as soon as some change occurred
• There was no way to know if we were viewing the offline or the online version of a page
• There was no was to temporarily disable the offline pages
• It implied hacking the Mediawiki code

So I rewrote it as a Mediawiki extension and tried to overcome the mentioned problems.
Note that it is done for the default style MonoBook.

Documentation

Bits I found useful for this task:

• the original idea
• http://code.google.com/apis/gears/api_localserver.html
• The Google Gears inspector:

svn checkout http://google-gears.googlecode.com/svn/trunk/gears/inspector google-gears-read-only

• http://www.mediawiki.org/wiki/Manual:MediaWiki_hooks

Google Gears browser extension

You must first install the plugin available here, note that it does a stupid browser detection and refuses my Iceweasel 2.0 so I had to cheat with my User Agent Switch extension and mimick Firefox 2.0 ;-)

Google Gears LocalServer extension

/extensions/GGears/

• Create a folder GGears in your MediaWiki extensions directory
• As I'm using rewriting rules, I had to add to /etc/apache2/sites-available/wiki.yobi.be:

RewriteCond %{REQUEST_URI} !/GGears/

gears_init.js

• Download the gears_init.js from the Google Gears web site and place it in the newly created folder

localserver.js

• Create localserver.js:

// Change this to set the name of the managed resource store to create.
// You use the name with the createManagedStore, and removeManagedStore,
// and openManagedStore APIs. It isn't visible to the user.
var STORE_NAME = "yobiwiki";

// Change this to set the URL of tha manifest file, which describe which
// URLs to capture. It can be relative to the current page, or an
// absolute URL.
var MANIFEST_FILENAME = "/extensions/GGears/manifest.php";

var localServer;
var store;
var blink=false;

// Called onload to initialize local server and store variables
function initStore() {
  if (!window.google || !google.gears) {
      hideItem("#p-ggears .portlet");
//    textOut("Google Gears not found");
  } else {
    localServer = google.gears.factory.create("beta.localserver");
    store = localServer.openManagedStore(STORE_NAME);
    if (!store) {
      textOut("No store found");
    } else {
      // We pass the current version to our manifest
      // php script to get only the recent editions
      store.manifestUrl = MANIFEST_FILENAME + '?since=' + store.currentVersion;
      textOut("");
    }
    menuStore();
  }
}

// Create (or open&update) the managed resource store
function createStore() {
  if (!window.google || !google.gears) {
    alert("You must install Google Gears first.");
    return;
  }
  store = localServer.createManagedStore(STORE_NAME);
  if (!store) {
    textOut("Error creating store");
  } else {
    store.manifestUrl = MANIFEST_FILENAME + '?since=' + store.currentVersion;
    textOut("Downloading...");
    hideItem("t-createStore");
    showItem("t-removeStore");
  }
  updateStore();
}

// Update the managed resource store
function updateStore() {
  if (!window.google || !google.gears) {
    alert("You must install Google Gears first.");
    return;
  }

  store.checkForUpdate();

  var timerId = window.setInterval(function() {
    // When the currentVersion property has a value, all of the resources
    // listed in the manifest file for that version are captured. There is
    // an open bug to surface this state change as an event.
    if (store.currentVersion) {
      window.clearInterval(timerId);
      textOut("Sync done!\n" +
              "Version: " +
              store.currentVersion + "\n" +
              "Refreshing...");
      menuStore();
      setTimeout("location.reload(true)", 2000);
    } else if (store.updateStatus == 3) {
      window.clearInterval(timerId);
      textOut("Error: " + store.lastErrorMessage);
    } else {
      textBlink();
    }
  }, 500);
}

// Remove the managed resource store.
function removeStore() {
  if (!window.google || !google.gears || !store) {
    return;
  }
  localServer.removeManagedStore(STORE_NAME);
  textOut("Store removed\n" +
          "Refreshing...");
  hideItem("t-createStore");
  hideItem("t-updateStore");
  hideItem("t-removeStore");
  hideItem("t-enableStore");
  hideItem("t-disableStore");
  setTimeout("location.reload(true)", 2000);
}

function enableStore() {
  if (!window.google || !google.gears || !store) {
    return;
  }
  store.enabled = true;
  menuStore();
  setTimeout("location.reload(true)", 5);
}

function disableStore() {
  if (!window.google || !google.gears || !store) {
    return;
  }
  store.enabled = false;
  menuStore();
  setTimeout("location.reload(true)", 5);
}

// Utility function to output some status text.
function textOut(s) {
 var elm = document.getElementById("t-textStore");
  while (elm.firstChild) {
    elm.removeChild(elm.firstChild);
  } 
  elm.appendChild(document.createTextNode(s));
  elm.style.fontWeight='normal';
  blink=false;
}

function textBlink() {
 var elm = document.getElementById("t-textStore");
  if (blink) {
    elm.style.fontWeight='normal';
    blink=false;
  } else {
    elm.style.fontWeight='bold';
    blink=true;
  }
}

// update the menu entries according to the current state
function menuStore() {
  if (!store) {
    showItem("t-createStore");
    hideItem("t-updateStore");
    hideItem("t-removeStore");
    hideItem("t-enableStore");
    hideItem("t-disableStore");
  } else {
    hideItem("t-createStore");
    showItem("t-updateStore");
    showItem("t-removeStore");
    if (store.enabled) {
      hideItem("t-enableStore");
      showItem("t-disableStore");
    } else {
      hideItem("t-disableStore");
      showItem("t-enableStore");
    }
  }
}

function showItem(s) {
 var elm = document.getElementById(s);
  elm.style.display='';
}

function hideItem(s) {
 var elm = document.getElementById(s);
  elm.style.display='none';
}

• Modify the store name in this localserver.js to what you want:

var STORE_NAME = "mymediawiki";

manifest.php

• Create the manifest.php that will create the file list on the fly (based on access to the database):

{
  "betaManifestVersion": 1,
<?php
require_once("../../DBSettings.php");

$l = mysql_connect($wgDBserver,$wgDBuser,$wgDBpassword);
mysql_selectdb($wgDBname);
$q = "SELECT max(rc_timestamp) from ".$wgDBprefix."recentchanges";
$r = mysql_query($q);
$t = mysql_fetch_row($r);
echo '  "version": "' . $t[0] . 'a",';
?>

  "entries": [
<?php
$q = "SELECT page_title FROM ".$wgDBprefix."page;";
$r = mysql_query($q);
while ($t = mysql_fetch_row($r))
{
    echo '    { "url": "/wiki/' . $t[0] . '" },'."\n";
}
?>
    { "url": "/skins/common/shared.css?97"},
    { "url": "/skins/common/commonPrint.css?97"},
    { "url": "/skins/common/wikibits.js?97"},
    { "url": "/skins/common/ajax.js?97"},
    { "url": "/skins/common/ajaxsearch.js?97"},
    { "url": "/skins/common/ajaxwatch.js?97"},
    { "url": "/skins/common/images/poweredby_mediawiki_88x31.png"},
    { "url": "/skins/common/images/gnu-fdl.png"},
    { "url": "/skins/monobook/main.css?97"},
    { "url": "/skins/monobook/headbg.jpg"},
    { "url": "/skins/monobook/external.png"},
    { "url": "/skins/monobook/bullet.gif"},
    { "url": "/skins/monobook/user.gif"},
    { "url": "/index.php?title=-&action=raw&smaxage=0&gen=js&useskin=monobook"},
    { "url": "/index.php?title=MediaWiki:Common.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"},
    { "url": "/index.php?title=MediaWiki:Monobook.css&usemsgcache=yes&action=raw&ctype=text/css&smaxage=18000"},
    { "url": "/index.php?title=-&action=raw&gen=css&maxage=18000&smaxage=0"},
    { "url": "/local/yobiwiki-pub.png"},
    { "url": "/extensions/GGears/localserver.js"},
    { "url": "/extensions/GGears/gears_init.js"}
  ]
}

Note that I moved my DB settings from LocalSettings.php to DBSettings.php, replaced now by a

require_once("DBSettings.php");

Otherwise you've to duplicate the following vars in your manifest.php:

$wgDBserver
$wgDBuser
$wgDBpassword
$wgDBname
$wgDBprefix

localserver.php

• Create the localserver.php as our Mediawiki extension:

<?php

if (defined('MEDIAWIKI')) {


function fnMyGGearsHookStyle() {
    return <<<EOS
<style type='text/css'>
#t-offStore {
  color: #7d7d7d;
}
#t-textStore {
  color: #ba0000;
}
</style>
EOS;
}

function fnMyGGearsHookJS($out) {
    $gearjs='<script type="text/javascript" src="/extensions/GGears/gears_init.js"></script>';
    $out->addScript($gearjs);
    $gearjs='<script type="text/javascript" src="/extensions/GGears/localserver.js"></script>';
    $out->addScript($gearjs);
    $out->addHeadItem('myggearsstyle', fnMyGGearsHookStyle());
    return true;
}

function fnMyGGearsHookBox() {
    echo "\n".'</ul>'."\n";
    echo '</div>'."\n";
    echo '</div>'."\n";
    echo '<div class="portlet" id="p-ggears">'."\n";
    echo '<h5>Gears LocalServer</h5>'."\n";
    echo '<div id="t-ggears" class="pBody">'."\n";
    echo '<ul>'."\n";
    echo '<li id="t-createStore"><a href="#" onclick="createStore();">create Store</a></li>'."\n";
    echo '<li id="t-updateStore"><a href="#" onclick="createStore();">update Store</a></li>'."\n";
    echo '<li id="t-enableStore"><a href="#" onclick="enableStore();">enable Store</a></li>'."\n";
    echo '<li id="t-disableStore"><a href="#" onclick="disableStore();">disable Store</a></li>'."\n";
    echo '<li id="t-removeStore"><a href="#" onclick="removeStore();">remove Store</a></li>'."\n";
    echo '<span id="t-offStore">'."\n";
    if (isset($_GET['off'])) {
        define('GGearsOffline');
        echo 'OFFLINE copy<br>'."\n";
    }
    echo '</span>'."\n";
    echo '<span id="t-textStore"></span>'."\n";
    echo '<script type="text/javascript">/*<![CDATA[*/ initStore(); /*]]>*/</script>'."\n";
    return true;
}

$wgExtensionCredits['other'][] = array('name' => 'GGears',
                           'version' => '0.1',
                           'author' => 'Philippe Teuwen',
//                           'url' => 'http://www.mediawiki.org/wiki/Extension:LocalServer',
                           'url' => 'http://wiki.yobi.be/Mediawiki_LocalServer',
                           'description' => 'Allows Google Gears to work offline');
$wgHooks['BeforePageDisplay'][] = 'fnMyGGearsHookJS';
$wgHooks['MonoBookTemplateToolboxEnd'][] = 'fnMyGGearsHookBox';
}

?>

• At the end of LocalSettings.php, add a link to the new extension:

require_once("$IP/extensions/GGears/localserver.php");

What is missing

What is crudely missing in the Google Gears API is a way to avoid to fetch all the pages at each new revision of the manifest.
Initially I made a manifest which accepts as argument the current version of the local Store and produces a list of the pages new or changed since.
But then GGears deletes all the other pages from the Store :-(

What I'd like to see:

• One version per URL instead of one per manifest
  • Very clean solution, manifest independent of the client state
• Or sth like that: given the version of the client Store the manifest could be generated on-the-fly with some:

{ "url": "/path/to/page","refresh": false }

Just in case, here is the manifest.php I tried to make with those assumptions in mind:

{
  "betaManifestVersion": 1,
<?php
require_once("../../DBSettings.php");
// We can get the version of the current DB to output only the
// pages changes since then, otherwise we dump all pages
$since = 0;
if (isset($_GET['since']) && is_numeric($_GET['since'])) {
    $since = $_GET['since'];
}
//error_log(date(DATE_RFC822).": since: ". $since . "\n", 3, './log.txt');

$l = mysql_connect($wgDBserver,$wgDBuser,$wgDBpassword);
mysql_selectdb($wgDBname);
$q = "SELECT max(rc_timestamp) from ".$wgDBprefix."recentchanges";
$r = mysql_query($q);
$t = mysql_fetch_row($r);
echo '  "version": "' . $t[0] . 'a",';
?>

  "entries": [
<?php
$q = "SELECT page_title FROM ".$wgDBprefix."page WHERE page_touched > ".$since.";";
$r = mysql_query($q);
while ($t = mysql_fetch_row($r))
{
    echo '    { "url": "/wiki/' . $t[0] . '" },'."\n";
}

// Just removing unchanged pages is not ok as they will be removed from the Store :-(
$q = "SELECT page_title FROM ".$wgDBprefix."page WHERE page_touched <= ".$since.";";
$r = mysql_query($q);
while ($t = mysql_fetch_row($r))
{
// Ideally I'd like to see sth like that to tell to keep the current page from the Store:
//    echo '    { "url": "/wiki/' . $t[0] . '","refresh": false },'."\n";
// But since the option doesn't exist in GGears, we've to fetch all pages again...
    echo '    { "url": "/wiki/' . $t[0] . '" },'."\n";
}
// Add also all the static ressources: css, images, js,...
?>
    { "url": "/extensions/GGears/localserver.js"},
    { "url": "/extensions/GGears/gears_init.js"}
  ]
}

With in initStore() and createStore() (localserver.js):

store.manifestUrl = MANIFEST_FILENAME + '?since=' + store.currentVersion;

So the current solution cannot avoid fetching all pages, the best we achieved was to returning a HTTP/1.0 304 Not Modified as often as possible.
This still requires about 30s to query 200 of my wiki pages (instead of 85s + some bandwidth for a real download)

TODO

auto refresh on save action

I've the impression this deletes the other pages whenever manifest mentions only the new ones :-(
Did I do all that for nothing ??

see Google Gears + Greasemonkey to take Wikipedia offline

see this discussion