niche: add the templates and static files

Pull back the cache time to reduce the dirty impact.

.gitignore

@@ -2,3 +2,4 @@
 *.pyc
 ignore/
 sessions/
+version.py

NOTES.txt

@@ -18,3 +18,31 @@ mysql -u niche -p niche < schema.sql
 Sanitizing
 ----------
 bleach looks fine https://github.com/jsocol/bleach
+
+Performance
+-----------
+All with niche-dev running a local server.
+
+base: 1592 ms
+web.config.debug = False: 841 ms
+Pre-compiled templates: 848 ms
+Basic MySQL caching: ~366 ms
+
+cProfile doesn't do multiple threads.
+
+HTML templates: 366 ms
+Precompiled templates: 360 ms (no advantage)
+
+Baseline: 366 ms
+Basic caching: 228 ms
+Binary pickle: 221 ms
+
+Before launch
+-------------
+Error and access logging: DONE
+Check /static/ is served by nginx: DONE
+Check gzip effects: DONE
+Way of logging bytes transferred: 
+error.log has no SQL in it: DONE
+memcached in monit: DONE
+monit emails sending: DONE

niche.py

@@ -1,610 +0,0 @@
-#!/usr/bin/python
-
-import datetime
-import hashlib
-import ConfigParser
-import passlib
-import re
-import time
-import subprocess
-
-import web
-import bleach
-from passlib.apps import custom_app_context as pwd_context
-
-import strings
-import utils
-
-urls = (
-    '/?', 'index',
-    '/links(/\d+)?(/\d+)?(/\d+)?', 'links',
-    '/link/new', 'new_link',
-    '/link/(\d+)', 'link',
-    '/link/(\d+)/hide', 'hide_link',
-    '/link/(\d+)/close', 'close_link',
-    '/link/(\d+)/new', 'new_comment',
-    '/comment/(\d+)/delete', 'delete_comment',
-    '/comment/(\d+)/like', 'like_comment',
-    '/user/([^/]+)', 'user',
-    '/user/([^/]+)/links', 'user_links',
-    '/user/([^/]+)/comments', 'user_comments',
-    '/user/([^/]+)/password', 'password',
-    '/login', 'login',
-    '/logout', 'logout',
-    '/newuser', 'newuser',
-)
-
-# Default configuration
-DEFAULTS = [
-    ( 'general', {
-            'dateformat': '%B %d, %Y',
-            'base': '/',
-            'wsgi': 'false',
-            'limit': 50,
-            }),
-    ( 'groups', {
-            'admins': '',
-            }),
-    ( 'db', {
-            'db': 'niche',
-            'user': 'niche',
-            'password': 'whatever',
-            }),
-    ( 'site', {
-            'name': 'Nichefilter',
-            'subtitle': 'of no fixed subtitle',
-            }),
-    ]
-
-def read_config():
-    """Set up the defaults and read in niche.ini, if any."""
-    cfg = ConfigParser.RawConfigParser()
-
-    for section, items in DEFAULTS:
-        cfg.add_section(section)
-
-        for name, value in items.items():
-            cfg.set(section, name, value)
-
-    cfg.read('niche.ini')
-    return cfg
-
-config = read_config()
-
-FEATURES = 'likes gravatar'.split()
-
-def get_features(config):
-    features = web.utils.Storage()
-
-    for feature in FEATURES:
-        features[feature] = config.has_option('features', feature) and config.getboolean('features', feature)
-
-    return features
-
-features = get_features(config)
-
-def get_version():
-    return subprocess.check_output('git describe --always --dirty'.split()).strip()
-
-def get_string(id):
-    """Get a string gettext style.  Splits the strings from the
-    code.
-    """
-    id = id.lower().replace(' ', '_').replace("'", "")
-    return strings.__dict__[id]
-
-_ = get_string
-
-db = web.database(dbn='mysql',
-                  user=config.get('db', 'user'),
-                  pw=config.get('db', 'password'),
-                  db=config.get('db','db'),
-                  )
-
-def require_feature(name):
-    if not features[name]:
-        raise web.notfound()
-
-def now():
-    return time.time()
-
-fallbacks = {
-    'user': web.utils.Storage(username='anonymous'),
-}
-
-class AutoMapper:
-    def __init__(self, type, around):
-        self._type = type
-        self._around = around
-
-    def __getattr__(self, name):
-        if hasattr(self._around, name):
-            return getattr(self._around, name)
-
-        key = '%sID' % name
-
-        if hasattr(self._around, key):
-            id = getattr(self._around, key)
-            got = first_or_none(name, key, id)
-
-            if got:
-                return got
-
-            if name in fallbacks:
-                return fallbacks[name]
-
-            raise web.notfound()
-
-        if name.endswith('s'):
-            singular = name[:-1]
-            table = '1_%s' % name
-            assert self._type
-            key = '%sID' % self._type
-
-            rows = db.select(table, where='%s = $id' % key, vars={'id': getattr(self, key)})
-            return [AutoMapper(singular, x) for x in rows]
-
-        raise AttributeError(name)
-
-    def ago(self):
-        return utils.ago(self.timestamp)
-
-    def to_date(self):
-        """Convert a timestamp to a date object."""
-        return datetime.date.fromtimestamp(self.timestamp)
-
-    def to_datestr(self):
-        """Convert a timestamp to a date string."""
-        return self.to_date().strftime(config.get('general', 'dateformat'))
-
-    def to_date_link(self):
-        """Convert a timestamp to a link."""
-        date = self.to_date()
-        return '%04d/%02d/%02d' % (date.year, date.month, date.day)
-
-def first_or_none(type, column, id, strict=False):
-    """Get the first item in the table that matches or None if there's
-    no match.
-    """
-    table = '1_%ss' % type
-    vs = db.select(table, where='%s = $id' % column, vars={'id': id}, limit=1)
-
-    if len(vs):
-        return AutoMapper(type, vs[0])
-    elif strict:
-        raise web.notfound()
-    else:
-        return None
-
-def first(type, column, id):
-    """Get the first matching item in the table or raise not found."""
-    return first_or_none(type, column, id, strict=True)
-
-class Model:
-    """Top level helpers.  Exposed to scripts."""
-    def is_admin(self):
-        id = session.get('userID', None)
-        return id != None and (str(id) in config.get('groups', 'admins').split())
-
-    def get_link(self, id):
-        """Get a link by link ID"""
-        return first_or_none('link', 'linkID', id)
-
-    def get_comment(self, id):
-        """Get a comment by comment ID"""
-        return first_or_none('comment', 'commentID', id, strict=True)
-
-    def get_user(self, id):
-        """Get a user by user ID"""
-        return first_or_none('user', 'userID', id)
-
-    def get_gravatar(self, email):
-        """Get the gravatar hash for an email"""
-        return hashlib.md5(email.strip().lower()).hexdigest()
-
-    def get_message(self):
-        """Get the message for the user, if any, and clear"""
-        message = session.get('message', None)
-
-        if message:
-            session.message = None
-
-        return message
-
-    def inform(self, message):
-        """Log a message to show the user on the next page"""
-        session.message = message
-
-    def get_active(self):
-        """Get the user entry for the currently logged in user, or
-        None.
-        """
-        id = session.get('userID', None)
-
-        if not id:
-            return None
-
-        return first_or_none('user', 'userID', id)
-
-model = Model()
-
-render = web.template.render(
-    'templates/',
-    base='layout',
-    globals={
-        'model': model,
-        'config': config,
-        'features': features,
-        'version': get_version(),
-        },
-    )
-
-app = web.application(urls, locals())
-
-def make_session():
-    """Helper that makes the session object, even if in debug mode."""
-    if web.config.get('_session') is None:
-        session = web.session.Session(app, web.session.DiskStore('sessions'),
-                                      initializer={'message': None}
-                                      )
-        web.config._session = session
-    else:
-        session = web.config._session
-
-    return session
-
-session = make_session()
-
-# Validate a password.  Pretty lax.
-password_validator = web.form.Validator(_("Short password"), lambda x: len(x) >= 3)
-
-def url_validator(v):
-    if not v:
-        return True
-
-    return re.match('(http|https|ftp|mailto)://.+', v)
-
-def redirect(url):
-    """Bounce to a different site absolute URL."""
-    raise web.seeother(url)
-
-def authenticate(msg=_("Login required")):
-    if not session.get('userID', None):
-        model.inform(msg)
-        redirect('/login')            
-
-def need_admin(msg):
-    if not model.is_admin():
-        model.inform(msg)
-        redirect('/login')            
-
-def error(message, condition, target='/'):
-    """Log an error if condition is true and bounce to somewhere."""
-    if condition:
-        model.inform(message)
-        redirect(target)
-
-def render_input(v):
-    """Tidy up user input and insert breaks for empty lines."""
-    v = bleach.clean(v)
-    
-    out = ''
-
-    for line in v.split('\n'):
-        if not line.strip():
-            out += '<br/>\n'
-        else:
-            out += line + '\n'
-
-    return out
-
-def render_links(where=None, span=None, vars={}):
-    input = web.input()
-    offset = int(input.get('offset', 0))
-    offset = max(offset, 0)
-
-    limit = int(input.get('limit', config.get('general', 'limit')))
-    limit = max(0, min(200, limit))
-
-    links = db.select('1_links', where=where, vars=vars, limit=limit, offset=offset, order="timestamp DESC")
-
-    if where:
-        results = db.query("SELECT COUNT(*) AS total FROM 1_links WHERE %s" % where, vars=vars)
-    else:
-        results = db.query("SELECT COUNT(*) AS total FROM 1_links")
-
-    total = results[0].total
-
-    return render.links([AutoMapper('link', x) for x in links], span, web.ctx.path, offset, limit, total)
-    
-class index:
-    def GET(self):
-        return render_links()
-
-class links:
-    def GET(self, year, month, day):
-        def tidy(v, low, high):
-            """Turn an optional parameter into a validated number"""
-            if v:
-                v = int(v[1:])
-                if v < low or v > high:
-                    raise web.notfound()
-
-                return False, v
-            else:
-                return True, low
-
-        no_year, year = tidy(year, 1990, 2037)
-        no_month, month = tidy(month, 1, 12)
-        no_day, day = tidy(day, 1, 31)
-
-        start = datetime.date(year, month, day)
-        span = None
-
-        # Figure out the span based on what was supplied
-        if no_year:
-            end = datetime.date(year + 100, month, day)
-            span = 'years'
-        elif no_month:
-            end = datetime.date(year + 1, month, day)
-            span = 'months'
-        elif no_day:
-            if month == 12:
-                end = datetime.date(year + 1, 1, day)
-            else:
-                end = datetime.date(year, month + 1, day)
-        else:
-            end = start + datetime.timedelta(days=1)
-
-        tstart = time.mktime(start.timetuple())
-        tend = time.mktime(end.timetuple())
-
-        return render_links(where='timestamp >= $tstart and timestamp < $tend', 
-                          vars={ 'tstart': tstart, 'tend': tend }, span=span)
-
-class link:
-    def GET(self, id):
-        link = model.get_link(id)
-        return render.link(link, None, False)
-
-class new_link:
-    form = web.form.Form(
-        web.form.Textbox('title', web.form.notnull),
-        web.form.Textbox('url', web.form.Validator(_("Not a URL"), url_validator)),
-        web.form.Textbox('url_description'),
-        web.form.Textarea('description', rows=10, cols=80),
-        web.form.Textarea('extended', rows=10, cols=80),
-        validators = [
-            web.form.Validator(_("URLs need a description"), lambda x: x.url_description if x.url else True),
-            web.form.Validator(_("Need a URL or description"), lambda x: x.url or x.description),
-            ]
-        )
-
-    def authenticate(self):
-        authenticate(_("Login to post"))
-
-    def GET(self):
-        self.authenticate()
-        return render.new_link(self.form(), None)
-
-    def POST(self):
-        self.authenticate()
-
-        form = self.form()
-
-        if not form.validates():
-            return render.new_link(form, None)
-
-        user = model.get_active()
-        url_description = render_input(form.d.url_description)
-        description = render_input(form.d.description)
-        extended = render_input(form.d.extended)
-
-        if 'preview' in web.input():
-            preview = web.utils.Storage(
-                title=form.d.title,
-                URL=form.d.url,
-                URL_description=url_description,
-                description=description,
-                extended=extended)
-
-            return render.new_link(form, preview)
-
-        next = db.insert('1_links',
-                         userID=user.userID,
-                         timestamp=now(),
-                         title=form.d.title,
-                         URL=form.d.url,
-                         URL_description=url_description,
-                         description=description,
-                         extended=extended
-                         )
-
-        model.inform(_("New post success"))
-        redirect('/link/%d' % next)
-
-class hide_link:
-    def GET(self, id):
-        link = model.get_link(id)
-        need_admin(_('Admin needed to hide a link'))
-
-        next = not link.hidden
-        db.update('1_links', where='linkID = $id', hidden=next, vars={'id': id})
-
-        model.inform(_("Link is hidden") if next else _("Link now shows"))
-        redirect('/link/%s' % id)
-
-class close_link:
-    def GET(self, id):
-        link = model.get_link(id)
-
-        need_admin(_('Admin needed to close a link'))
-        next = not link.closed
-        db.update('1_links', where='linkID = $id', closed=next, vars={'id': id})
-
-        model.inform(_("Link is closed") if next else _("Link is open"))
-        redirect('/link/%s' % id)
-
-class new_comment:
-    form = web.form.Form(
-        web.form.Textarea('content', web.form.notnull, rows=10, cols=80)
-        )
-
-    def check(self, id):
-        authenticate(_("Login to comment"))
-        link = model.get_link(id)
-
-        error(_("Link is closed"), link.closed)
-
-        return link
-
-    def GET(self, id):
-        link = self.check(id)
-        return render.link(link, self.form(), None)
-
-    def POST(self, id):
-        link = self.check(id)
-        form = self.form()
-
-        if not form.validates():
-            return render.link(link, form, None)
-
-        user = model.get_active()
-        content = render_input(form.d.content)
-
-        if 'preview' in web.input():
-            return render.link(link, form, content)
-
-        next = db.insert('1_comments',
-                         linkID=link.linkID,
-                         userID=user.userID,
-                         timestamp=now(),
-                         content=content
-                         )
-
-        model.inform(_("New comment success"))
-        redirect('/link/%d' % link.linkID)
-
-class delete_comment:
-    def GET(self, id):
-        comment = model.get_comment(id)
-
-        need_admin(_('Admin needed to delete a comment'))
-        db.delete('1_comments', where='commentID = $id', vars={'id': id})
-
-        model.inform(_("Comment deleted"))
-        redirect('/link/%s' % comment.linkID)
-
-class like_comment:
-    def GET(self, id):
-        check_feature('likes')
-        authenticate(_("Login to like"))
-        comment = model.get_comment(id)
-
-        userID = session.userID
-        db.insert('1_likes', commentID=comment.commentID, userID=userID)
-
-        model.inform(_("Liked"))
-        redirect('/link/%s' % comment.linkID)
-
-class user:
-    def GET(self, id):
-        user = first('user', 'username', id)
-        return render.user(user)
-
-class user_links:
-    def GET(self, id):
-        user = first('user', 'username', id)
-        return render_links(where='userID=$id', vars={'id': user.userID})
-
-class user_comments:
-    def GET(self, id):
-        user = first('user', 'username', id)
-        comments = db.select('1_comments', where='userID=$id', order='timestamp DESC',
-                             vars={'id': user.userID},
-                             limit=config.get('general', 'limit'))
-        return render.user_comments([AutoMapper('comment', x) for x in comments])
-
-class login:
-    login = web.form.Form(
-        web.form.Textbox('username', web.form.notnull),
-        web.form.Password('password', web.form.notnull),
-        )
-
-    def GET(self):
-        return render.login(self.login())
-
-    def POST(self):
-        form = self.login()
-
-        if not form.validates():
-            return render.login(form)
-
-        user = first_or_none('user', 'username', form.d.username)
-        ok = False
-
-        if user:
-            try:
-                ok = passlib.hash.mysql323.verify(form.d.password, user.password)
-            except ValueError:
-                ok = pwd_context.verify(form.d.password, user.password)
-
-        if not ok:
-            form.valid = False
-            model.inform(_("Bad username or password"))
-            return render.login(form)
-
-        session.userID = user.userID
-
-        model.inform(_("Logged in"))
-        redirect('/')
-
-class logout:
-    def GET(self):
-        session.userID = None
-
-        model.inform(_("Logged out"))
-        redirect('/')
-
-class password:
-    form = web.form.Form(
-        web.form.Password('password', web.form.notnull, password_validator,
-                          description=_("New password")),
-        web.form.Password('again', web.form.notnull, description=_("Password again")),
-        validators=[
-            web.form.Validator(_("Passwords don't match"), lambda x: x.password == x.again)
-            ]
-        )
-
-    def authenticate(self, id):
-        authenticate()
-
-        active = model.get_active()
-        error(_("Permission denied"), active.username != id, '/user/%s' % id)
-
-    def GET(self, id):
-        self.authenticate(id)
-        return render.password(self.form())
-
-    def POST(self, id):
-        self.authenticate(id)
-
-        form = self.form()
-
-        if not form.validates():
-            return render.login(form)
-
-        db.update('1_users', password=pwd_context.encrypt(form.d.password), where='username=$id', vars={'id': id})
-        
-        model.inform(_("Password changed"))
-        redirect('/user/%s' % id)
-
-if __name__ == "__main__":
-    if config.getboolean('general', 'wsgi'):
-        web.config.debug = False
-        web.wsgi.runwsgi = lambda func, addr=None: web.wsgi.runfcgi(func, addr)
-    else:
-        # Development machine.  Run stand alone
-        pass
-
-    app.run()

niche/__main__.py

@@ -0,0 +1,4 @@
+from . import app
+
+if __name__ == "__main__":
+    app.main()

niche/strings.py

@@ -26,3 +26,29 @@ admin_needed_to_delete_a_comment = 'Admin needed to delete a comment'
 admin_needed_to_hide_a_link = 'Admin needed to hide a link'
 admin_needed_to_close_a_link = 'Admin needed to close a link'
 bad_username_or_password = "Bad username or password"
+only_the_user_can_checkout_their_links = "Only the user can checkout their links"
+possible_cross_site_request_forgery_try_again = "Possible cross site request forgery.  Try again."
+only_admins_can_access_debug_pages = "Only admins can access debug pages."
+field_realname = "Real name"
+field_email = "Email"
+field_gravatar_email = "Gravatar email"
+field_homepage = "Homepage"
+field_team = "Team"
+field_location = "Location"
+field_twitter = "Twitter"
+field_facebook = "Facebook"
+field_google_plus = "Google+"
+field_skype = "Skype"
+field_aim = "AIM"
+field_bio = "Bio"
+field_title = "Title"
+field_url = "URL"
+field_url_description = "URL description"
+field_extended = "Extended"
+field_description = "Description"
+field_use_markdown = "Use Markdown"
+field_comment = "Comment"
+field_username = "Username"
+field_password = "Password"
+field_new_password = "New password"
+field_again = "Again"

niche/test_niche.py

@@ -0,0 +1,2 @@
+def test_loads():
+    import niche

niche/utils.py

@@ -1,4 +1,30 @@
+import collections
+import copy
 import time
+import threading
+
+
+class Counters(object):
+    def __init__(self):
+        self.counters = collections.defaultdict(int)
+        self.lock = threading.Lock()
+        self.counters['id'] = id(self)
+
+    def bump(self, name, arg=None):
+        if not isinstance(name, str):
+            name = name.__class__.__name__
+            if '.' in name:
+                name = name[name.rindex('.')+1:]
+        if arg:
+            name = '%s/%s' % (name, arg)
+
+        with self.lock:
+            self.counters[name] += 1
+
+    def get_snapshot(self):
+        with self.lock:
+            return copy.copy(self.counters)
+
 
 def now():
     """Return the current time in the right epoch."""

niche/version.py

@@ -0,0 +1 @@
+__version__ = 'r1-5-gb99a317-dirty'

schema.sql

@@ -144,6 +144,7 @@ CREATE TABLE `1_users` (
   `bio` mediumtext NOT NULL,
   `preferences` text,
   `last_visit` int(14) unsigned NOT NULL default '0',
+  `contacts` text,
   PRIMARY KEY  (`userID`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 

setup.py

@@ -0,0 +1,30 @@
+import glob
+
+from setuptools import find_packages, setup
+
+setup(
+    name='niche',
+    version_format='{tag}.dev{commitcount}+{gitsha}',
+    setup_requires=['setuptools-git-version'],
+    url='https://juju.nz/src/juju/niche',
+    author='Michael Hope',
+    author_email='michaelh@juju.nz',
+    description='A community web log for monkey niches.',
+    zip_safe=False,
+    packages=find_packages(),
+    data_files=[
+        ('lib/niche/templates', glob.glob('templates/*')),
+        ('lib/niche/static/themes/common',
+         glob.glob('static/themes/common/*')),
+        ('lib/niche/static/themes/able-wpcom',
+         glob.glob('static/themes/able-wpcom/*')),
+    ],
+    install_requires=[
+        'passlib>=1.7.4',
+        'bleach>=3.2.1',
+        'markdown>=3.1.1',
+        'python-memcached>=1.59',
+        'web.py>=0.62',
+        'pymysql>=0.9.3',
+    ],
+)

static/style.css

@@ -1,54 +0,0 @@
-body {
-    font-family: Ubuntu, Tahoma, Geneva;
-}
-
-h1 {
-    background-color: yellow;
-    margin: 0;
-    padding: 0.3em;
-    border-bottom: 1px solid #B0B000;
-}
-
-h1 a {
-    color: black;
-    text-decoration: none;
-}
-
-.meta {
-    background-color: lightblue;
-    border: 1px solid blue;
-    padding: 0.5em;
-}
-
-.message {
-    background-color: lightgreen;
-    border: 1px solid green;
-    padding: 0.5em;
-}
-
-.error {
-    background-color: pink;
-    border: 1px solid red;
-    padding: 0.5em;
-}
-
-.wrong {
-}
-
-.preview {
-    background-color: #e0e0e0;
-    border: 1px solid #101010;
-    padding: 0.5em;
-}
-
-.link {
-    margin: 0em 0em 0.5em 1em;
-}
-
-.comment {
-    border-top: 1px solid #808080;
-}
-
-.byline {
-    font-size: small;
-}

static/themes/able-wpcom/html5.js

@@ -0,0 +1,288 @@
+/*! HTML5 Shiv v3.6.1 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed */
+;(function(window, document) {
+/*jshint evil:true */
+  /** Preset options */
+  var options = window.html5 || {};
+
+  /** Used to skip problem elements */
+  var reSkip = /^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i;
+
+  /** Not all elements can be cloned in IE **/
+  var saveClones = /^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i;
+
+  /** Detect whether the browser supports default html5 styles */
+  var supportsHtml5Styles;
+
+  /** Name of the expando, to work with multiple documents or to re-shiv one document */
+  var expando = '_html5shiv';
+
+  /** The id for the the documents expando */
+  var expanID = 0;
+
+  /** Cached data for each document */
+  var expandoData = {};
+
+  /** Detect whether the browser supports unknown elements */
+  var supportsUnknownElements;
+
+  (function() {
+    try {
+        var a = document.createElement('a');
+        a.innerHTML = '<xyz></xyz>';
+        //if the hidden property is implemented we can assume, that the browser supports basic HTML5 Styles
+        supportsHtml5Styles = ('hidden' in a);
+
+        supportsUnknownElements = a.childNodes.length == 1 || (function() {
+          // assign a false positive if unable to shiv
+          (document.createElement)('a');
+          var frag = document.createDocumentFragment();
+          return (
+            typeof frag.cloneNode == 'undefined' ||
+            typeof frag.createDocumentFragment == 'undefined' ||
+            typeof frag.createElement == 'undefined'
+          );
+        }());
+    } catch(e) {
+      supportsHtml5Styles = true;
+      supportsUnknownElements = true;
+    }
+
+  }());
+
+  /*--------------------------------------------------------------------------*/
+
+  /**
+   * Creates a style sheet with the given CSS text and adds it to the document.
+   * @private
+   * @param {Document} ownerDocument The document.
+   * @param {String} cssText The CSS text.
+   * @returns {StyleSheet} The style element.
+   */
+  function addStyleSheet(ownerDocument, cssText) {
+    var p = ownerDocument.createElement('p'),
+        parent = ownerDocument.getElementsByTagName('head')[0] || ownerDocument.documentElement;
+
+    p.innerHTML = 'x<style>' + cssText + '</style>';
+    return parent.insertBefore(p.lastChild, parent.firstChild);
+  }
+
+  /**
+   * Returns the value of `html5.elements` as an array.
+   * @private
+   * @returns {Array} An array of shived element node names.
+   */
+  function getElements() {
+    var elements = html5.elements;
+    return typeof elements == 'string' ? elements.split(' ') : elements;
+  }
+
+    /**
+   * Returns the data associated to the given document
+   * @private
+   * @param {Document} ownerDocument The document.
+   * @returns {Object} An object of data.
+   */
+  function getExpandoData(ownerDocument) {
+    var data = expandoData[ownerDocument[expando]];
+    if (!data) {
+        data = {};
+        expanID++;
+        ownerDocument[expando] = expanID;
+        expandoData[expanID] = data;
+    }
+    return data;
+  }
+
+  /**
+   * returns a shived element for the given nodeName and document
+   * @memberOf html5
+   * @param {String} nodeName name of the element
+   * @param {Document} ownerDocument The context document.
+   * @returns {Object} The shived element.
+   */
+  function createElement(nodeName, ownerDocument, data){
+    if (!ownerDocument) {
+        ownerDocument = document;
+    }
+    if(supportsUnknownElements){
+        return ownerDocument.createElement(nodeName);
+    }
+    if (!data) {
+        data = getExpandoData(ownerDocument);
+    }
+    var node;
+
+    if (data.cache[nodeName]) {
+        node = data.cache[nodeName].cloneNode();
+    } else if (saveClones.test(nodeName)) {
+        node = (data.cache[nodeName] = data.createElem(nodeName)).cloneNode();
+    } else {
+        node = data.createElem(nodeName);
+    }
+
+    // Avoid adding some elements to fragments in IE < 9 because
+    // * Attributes like `name` or `type` cannot be set/changed once an element
+    //   is inserted into a document/fragment
+    // * Link elements with `src` attributes that are inaccessible, as with
+    //   a 403 response, will cause the tab/window to crash
+    // * Script elements appended to fragments will execute when their `src`
+    //   or `text` property is set
+    return node.canHaveChildren && !reSkip.test(nodeName) ? data.frag.appendChild(node) : node;
+  }
+
+  /**
+   * returns a shived DocumentFragment for the given document
+   * @memberOf html5
+   * @param {Document} ownerDocument The context document.
+   * @returns {Object} The shived DocumentFragment.
+   */
+  function createDocumentFragment(ownerDocument, data){
+    if (!ownerDocument) {
+        ownerDocument = document;
+    }
+    if(supportsUnknownElements){
+        return ownerDocument.createDocumentFragment();
+    }
+    data = data || getExpandoData(ownerDocument);
+    var clone = data.frag.cloneNode(),
+        i = 0,
+        elems = getElements(),
+        l = elems.length;
+    for(;i<l;i++){
+        clone.createElement(elems[i]);
+    }
+    return clone;
+  }
+
+  /**
+   * Shivs the `createElement` and `createDocumentFragment` methods of the document.
+   * @private
+   * @param {Document|DocumentFragment} ownerDocument The document.
+   * @param {Object} data of the document.
+   */
+  function shivMethods(ownerDocument, data) {
+    if (!data.cache) {
+        data.cache = {};
+        data.createElem = ownerDocument.createElement;
+        data.createFrag = ownerDocument.createDocumentFragment;
+        data.frag = data.createFrag();
+    }
+
+
+    ownerDocument.createElement = function(nodeName) {
+      //abort shiv
+      if (!html5.shivMethods) {
+          return data.createElem(nodeName);
+      }
+      return createElement(nodeName, ownerDocument, data);
+    };
+
+    ownerDocument.createDocumentFragment = Function('h,f', 'return function(){' +
+      'var n=f.cloneNode(),c=n.createElement;' +
+      'h.shivMethods&&(' +
+        // unroll the `createElement` calls
+        getElements().join().replace(/\w+/g, function(nodeName) {
+          data.createElem(nodeName);
+          data.frag.createElement(nodeName);
+          return 'c("' + nodeName + '")';
+        }) +
+      ');return n}'
+    )(html5, data.frag);
+  }
+
+  /*--------------------------------------------------------------------------*/
+
+  /**
+   * Shivs the given document.
+   * @memberOf html5
+   * @param {Document} ownerDocument The document to shiv.
+   * @returns {Document} The shived document.
+   */
+  function shivDocument(ownerDocument) {
+    if (!ownerDocument) {
+        ownerDocument = document;
+    }
+    var data = getExpandoData(ownerDocument);
+
+    if (html5.shivCSS && !supportsHtml5Styles && !data.hasCSS) {
+      data.hasCSS = !!addStyleSheet(ownerDocument,
+        // corrects block display not defined in IE6/7/8/9
+        'article,aside,figcaption,figure,footer,header,hgroup,nav,section{display:block}' +
+        // adds styling not present in IE6/7/8/9
+        'mark{background:#FF0;color:#000}'
+      );
+    }
+    if (!supportsUnknownElements) {
+      shivMethods(ownerDocument, data);
+    }
+    return ownerDocument;
+  }
+
+  /*--------------------------------------------------------------------------*/
+
+  /**
+   * The `html5` object is exposed so that more elements can be shived and
+   * existing shiving can be detected on iframes.
+   * @type Object
+   * @example
+   *
+   * // options can be changed before the script is included
+   * html5 = { 'elements': 'mark section', 'shivCSS': false, 'shivMethods': false };
+   */
+  var html5 = {
+
+    /**
+     * An array or space separated string of node names of the elements to shiv.
+     * @memberOf html5
+     * @type Array|String
+     */
+    'elements': options.elements || 'abbr article aside audio bdi canvas data datalist details figcaption figure footer header hgroup mark meter nav output progress section summary time video',
+
+    /**
+     * A flag to indicate that the HTML5 style sheet should be inserted.
+     * @memberOf html5
+     * @type Boolean
+     */
+    'shivCSS': (options.shivCSS !== false),
+
+    /**
+     * Is equal to true if a browser supports creating unknown/HTML5 elements
+     * @memberOf html5
+     * @type boolean
+     */
+    'supportsUnknownElements': supportsUnknownElements,
+
+    /**
+     * A flag to indicate that the document's `createElement` and `createDocumentFragment`
+     * methods should be overwritten.
+     * @memberOf html5
+     * @type Boolean
+     */
+    'shivMethods': (options.shivMethods !== false),
+
+    /**
+     * A string to describe the type of `html5` object ("default" or "default print").
+     * @memberOf html5
+     * @type String
+     */
+    'type': 'default',
+
+    // shivs the document according to the specified `html5` object options
+    'shivDocument': shivDocument,
+
+    //creates a shived element
+    createElement: createElement,
+
+    //creates a shived documentFragment
+    createDocumentFragment: createDocumentFragment
+  };
+
+  /*--------------------------------------------------------------------------*/
+
+  // expose html5
+  window.html5 = html5;
+
+  // shiv the document
+  shivDocument(document);
+
+}(this, document));

static/themes/able-wpcom/style.css

@@ -1276,6 +1276,13 @@ article.comment {
 #commentform input[ type="text" ] {
 	display: block;
 }
+#commentform input[ type="password" ] {
+	display: block;
+}
+#commentform input[ type="checkbox" ] {
+    margin-left: 0.5em;
+}
+
 #commentform .required {
 	color: rgb( 255, 0, 0 ); /* #ff0000 */
 }

static/themes/common/niche.css

@@ -0,0 +1,36 @@
+.new {
+    background-color: lightyellow;
+    border: 1px solid yellow;
+    padding: 0.3em;
+    font-size: small;
+}
+
+.meta {
+    background-color: lightblue;
+    border: 1px solid blue;
+    padding: 0.3em;
+}
+
+.message {
+    background-color: lightgreen;
+    border: 1px solid green;
+    padding: 0.3em;
+}
+
+.error {
+    background-color: pink;
+    border: 1px solid red;
+    padding: 0.3em;
+}
+
+.wrong {
+    background-color: pink;
+    border: 1px solid red;
+    padding: 0.3em;
+}
+
+.preview {
+    background-color: #e0e0e0;
+    border: 1px solid #101010;
+    padding: 0.3em;
+}

templates/internal_gc.html

@@ -0,0 +1,8 @@
+$def with (title, diffs)
+
+<h1>$title</h1>
+<table border=1>
+<tr><th>Type</th><th>Count</th></tr>
+$for type, count in diffs:
+  <tr><td>$type</td><td>$count</td></tr>
+</table>

templates/layout.html

@@ -6,10 +6,16 @@ $ active = model.get_active()
 <html>
   <head>
     <meta name="viewport" content="width=device-width" />
+<!--[if lt IE 9]>
+    <script src="static/themes/able-wpcom/html5.js" type="text/javascript"></script>
+<![endif]-->
     <title>$config.get('site', 'name')</title>
     $ base = config.get('general', 'base')
     $if base: <base href="$base"/>
     <link rel="stylesheet" type="text/css" href="static/themes/able-wpcom/style.css">
+    <link rel="stylesheet" type="text/css" href="static/themes/common/niche.css">
+    $if features.rss:
+      <link rel="alternate" type="application/rss+xml" href="rss" title="RSS feed">
   </head>
   <body>
     <div id="page" class="hfeed site">
@@ -25,31 +31,58 @@ $ active = model.get_active()
               <li><a href="user/$active.username">$active.username</a>
               <li><a href="logout">Logout</a>
             $else:
-              <li><a href="login">Login</a>
+              <li>
+              $if config.get('general', 'has_https'):
+                <a href="https://$config.get('general', 'domain')${base}login">Login</a>
+              $else:
+                <li><a href="login">Login</a>
             <li><a href="link/new">New post</a>
             <li><a href="links">Archive</a>
             </ul>
           </nav>
         </header>
         <div id="main">
-          $ message = model.get_message()
-          $if message:
-            <p class="message">$message</p>
           <div id="primary" class="site-content">
+            $ message = model.get_message()
+            $if message:
+              <p class="message">$message</p>
             <div id="content" role="main">
               $:content
             </div>
           </div>
+          <div id="secondary">
+            $if active:
+              $ new = model.get_new()
+              $if new:
+                <div class="new">
+                <p>New comments on
+                <ul>
+                $for comment in new:
+                  $ link = comment.link
+                  <li><a href="link/$link.linkID#comment_$comment.commentID">
+                    $if not link.URL: $link.title
+                    $else: $:link.URL_description
+                  </a>
+                </ul>
+                </div>
+          </div>
         </div>
+          <hr/>
 	<footer id="colophon" class="site-footer" role="contentinfo">
           <div class="site-info">
             A <a href="https://juju.net.nz/src/niche.git/">niche</a> community filter
 	    <span class="sep"> | </span>
             Version $version
+            $if config.get('site', 'license'):
+	      <span class="sep"> | </span>
+              $:config.get('site', 'license')
 	    <span class="sep"> | </span>
             <a href="https://github.com/nzmichaelh/niche/issues">Bugs</a>
 	    <span class="sep"> | </span>
             Theme based on <a href="http://abledemo.wordpress.com/">Able</a>
+            $if features.rss:
+	      <span class="sep"> | </span>
+              <a type="application/rss+xml" href="rss"><img src="static/themes/common/feed-icon-14x14.png"/> RSS feed</a>
           </div>
         </footer>
       </div>

templates/link.html

@@ -3,16 +3,18 @@ $def with (link, form, preview)
 $ here = "link/%s" % link.linkID
 
 $ date = link.to_datestr()
-<h2 class="entry-title">
-  <a href="daily/$link.to_date_link()">$date</a>
-</h2>
+<h1 class="entry-title">
+  <a href="links/$link.to_date_link()">$date</a>
+</h1>
 
 $if model.is_admin():
   <p class="meta">
-    <a href="$here/hide">${ "Hidden" if link.hidden else "Showing" }</a> |
+    <a href="$here/hide">${ "Hidden" if link.hidden else "Showing" }</a>
+    <span class="sep"> | </span>
     <a href="$here/close">${ "Closed" if link.closed else "Open for comments" }</a>
   </p>
 
+<div class="entry-summary">
 $if not link.URL:
     <b>$link.title</b>
 $else:
@@ -20,43 +22,46 @@ $else:
 $:link.description
 $if link.extended:
     <p>$:link.extended
-<p class="byline">
+</div>
+<div class="entry-meta">
 $ user = link.user
 posted by <a href="user/$user.username">$user.username</a>
 $link.ago() ago
+</div>
 
 $ comments = link.comments
 
 <ul class="commentlist">
 $for comment in comments:
     <li>
-    <div class="comment">
+    <div class="comment" id="comment_$comment.commentID">
     $:comment.content
+    </b></i>
     <footer>
     $ user = comment.user
-    posted by <a href="user/$user.username">$user.username</a>
-    $link.ago() ago
+    posted <a href="$here#comment_$comment.commentID">$comment.ago()</a> ago
+    by <a href="user/$user.username">$user.username</a>
     $if features.likes:
-        $ likes = comment.likes
-        | $len(likes) likes
-        | <a href="comment/$comment.commentID/like">I like it</a>
+        <span class="sep"> | </span>
+        $model.plural(comment.like_count, 'like')
+        <span class="sep"> | </span>
+        <a href="comment/$comment.commentID/like">I like it</a>
     $if model.is_admin():
-        | <a href="comment/$comment.commentID/delete">Delete</a>
+        <span class="sep"> | </span>
+        <a href="comment/$comment.commentID/delete">Delete</a>
     </footer>
     </div>
 </ul>
 
 $if not link.closed:
-    $if form:
-        <a id="comment"></a>
-        $if preview != None:
-            <p class="preview">$:preview
-        <div id="commentform">
-        <form name="main" method="post"> 
-	  $:form.render()
-	  <input type="submit" name="post" value="Post"/>
-	  <input type="submit" name="preview" value="Preview"/>
-	</form>
-        </div>
-    $else:
-        <p><a href="/link/$link.linkID/new#comment">New comment</a>
+  <a id="comment"></a>
+  $if preview:
+    <div class="preview">$:preview</div><p>
+  <div id="commentform">
+    <form name="main" method="post" action="$here/new">
+      $:form.render_css()
+      <br/>
+      <input type="submit" name="post" value="Post"/>
+      <input type="submit" name="preview" value="Preview"/>
+    </form>
+  </div>

templates/links.html

@@ -1,4 +1,21 @@
-$def with (links, span, base, offset, limit, total)
+$def with (links, span, base, offset, limit, total, date_range)
+
+$if date_range:
+  $for year in date_range.years:
+    $if not loop.first: <span class="sep"> | </span>
+    $if year == date_range.year: <b>
+    <a href="links/$year">$year</a>
+    $if year == date_range.year: </b>
+  <br/>
+  $if date_range.month is None: <b>
+  <a href="links/$date_range.year">Whole year</a>
+  $if date_range.month is None: </b>
+  $for i in range(1, len(date_range.months)):
+      <span class="sep"> | </span>
+      $if i == date_range.month: <b>
+      <a href="links/$date_range.year/$i">$date_range.months[i]</a>
+      $if i == date_range.month: </b>
+  <hr/>
 
 $ last_date = None
 
@@ -12,7 +29,7 @@ $for link in links:
 
     <div class="entry-summary">
     $if not link.URL:
-        $link.title
+        <b>$link.title</b>
     $else:
         <a href="$link.URL">$:link.URL_description</a>
     $:link.description
@@ -20,16 +37,17 @@ $for link in links:
         <a href="link/$link.linkID" class="more-link">more inside</a>
     </div>
     <div class="entry-meta">
-      posted by <a href="user/$link.user.username">$link.user.username</a>
-      $link.ago() ago
+      posted $link.ago() ago
+      by <a href="user/$link.user.username">$link.user.username</a>
+      
       -
-      <a href="link/$link.linkID">$len(link.comments) comments</a>
+      <a href="link/$link.linkID">$model.plural(link.comment_count, 'comment')</a>
     </div>
 
-$if total > limit:
+$ at, pages = model.paginate(offset, total, limit)
+$if pages:
     Page
-    $ at = 1 + offset // limit
-    $for i in range(0, total, limit):
-        $if loop.index == at: <b>
-        <a href="$base?offset=$i">$loop.index</a>
-        $if loop.index == at: </b>
+    $for page in pages:
+        $if page == at: <b>
+        <a href="$base?offset=${(page-1)*limit}">$page</a>
+        $if page == at: </b>

templates/login.html

@@ -1,6 +1,15 @@
 $def with (form)
 
+<div id="commentform">
 <form name="main" method="post"> 
-$:form.render()
+$:form.render_css()
     <input type="submit"/>
 </form>
+</div>
+$ contact = config.get('site', 'contact')
+$if contact:
+   Forgotten your password?
+   <a href="mailto:$contact?subject=$config.get('site', 'name')%20password%20reset">
+     Send us an email.
+   </a>
+   <p/>

templates/new_comment.html

@@ -1,6 +0,0 @@
-$def with (link, form)
-
-<form name="main" method="post"> 
-$:form.render()
-    <input type="submit"/>
-</form>

templates/new_link.html

@@ -13,8 +13,12 @@ $if preview != None:
     </div>
 <p>
 
+<div id="commentform">
 <form name="main" method="post"> 
-$:form.render()
+$:form.render_css()
+<br/>
 <input type="submit" name="post" value="Post"/>
 <input type="submit" name="preview" value="Preview"/>
 </form>
+</div>
+

templates/password.html

@@ -1,8 +1,12 @@
 $def with (form)
 
+
+<div id="commentform">
 <form name="main" method="post"> 
 $if not form.valid:
     <p class="error">Something's wrong</p>
-$:form.render()
+$:form.render_css()
+    <br/>
     <input type="submit"/>
 </form>
+</div>

templates/rss.html

@@ -0,0 +1,32 @@
+$def with (links, base)
+<!--?xml version="1.0"?-->
+<rss version="2.0">
+   <channel>
+      <title>$config.get('site', 'name')</title>
+      <link>$base</link>
+      $ subtitle = config.get('site', 'subtitle')
+      $if subtitle: <description>$subtitle</description>
+      <language>en-us</language>
+      <generator>niche</generator>
+      $ contact = config.get('site', 'contact')
+      $if contact: <managingEditor>$contact</managingEditor>
+      $if contact: <webMaster>$contact</webMaster>
+
+      $for link in links:
+        <item>
+          <title>
+            $if not link.URL: $link.title
+            $else: $link.URL_description
+          </title>
+          <description>
+            $if link.URL:
+              <![CDATA[<a href="$link.URL">$:link.URL_description</a>]]>
+            <![CDATA[$:link.description]]>
+          </description>
+          <author>$link.user.username</author>
+          <pubDate>$model.to_rss_date(link.timestamp)</pubDate>
+          <link>$base/link/$link.linkID</link>
+          <guid>$base/link/$link.linkID</guid>
+        </item>
+    </channel>
+</rss>

templates/user.html

@@ -1,20 +1,34 @@
 $def with (user)
 
-<h2>$user.username's profile</h2>
-
-<p>
-$if features.gravatar:
-  <img src="http://www.gravatar.com/avatar/$model.get_gravatar(user.email)"/>
-Name: $user.realname
-<p>User ID: $user.userID
-$if model.get_active() and user.userID == model.get_active().userID:
+$ fields = user.contacts_json
+<h2>
+$if fields.gravatar_email:
+  <img src="http://www.gravatar.com/avatar/$model.get_gravatar(fields.gravatar_email)"/>
+$user.username (user #$user.userID)</h2>
+$if model.get_active():
+    <ul>
+    $for field in config.getlist('general', 'user_fields'):
+        $ value = fields.get(field)
+        $ value = value if value else user.get(field)
+        $if value:
+            <li>$model.field_text(field): $:linkify(value)
+    </ul>
+$if model.is_user_or_admin(user.userID):
+    <a href="user/$user.username/edit">Edit</a>
+    <span class="sep"> | </span>
     <a href="user/$user.username/password">Set password</a>
-<p>Homepage: <a href="$user.homepage">$user.homepage</a>
-<p>Bio:
-$:user.bio
-
+$else:
+    <a href="login">Log in</a> for details.
+<p><hr/>
+$:render_input(user.bio)
+<p><hr/>
 <p>
-  <a href="user/$user.username/links">$len(user.links) links</a>
-| <a href="user/$user.username/comments">$len(user.comments) comments</a>
+  <a href="user/$user.username/links">$model.plural(user.link_count, 'link')</a>
+  <span class="sep"> | </span>
+  <a href="user/$user.username/comments">$model.plural(user.comment_count, 'comment')</a>
+$if model.is_user_or_admin(user.userID):
+  <span class="sep"> | </span>
+  <a href="user/$user.username/checkout">All links as RSS</a>
 $if features.likes:
-  | <a href="user/$user.username/likes">$len(user.likes) likes</a>
+  <span class="sep"> | </span>
+  <a href="user/$user.username/likes">$model.plural(user.like_count, 'like')</a>

templates/user_edit.html

@@ -0,0 +1,12 @@
+$def with (user, form)
+
+<h2>Editing $user.username</h2>
+
+<div id="commentform">
+<form name="main" method="post">
+  $:form.render_css()
+  <br/>
+  <input type="submit" name="post" value="Update"/>
+</form>
+</div>
+<p>Your bio is shown to all visitors.  Other details are only shown to logged in users.

tests.mk

@@ -0,0 +1,15 @@
+GET = / \
+	/links /links/2010 /links/2010/12 \
+	/link/2 /link/100 \
+	/user/michaelh /user/tracicle \
+	/login \
+	/rss /rss.php /rss.xml
+
+BASE = http://ch.monkeyfilter.com
+
+ALL = $(GET:%=%-get)
+
+all: $(ALL)
+
+%-get:
+	wget -O /dev/null -nv $(BASE)$*

update.sql

@@ -0,0 +1 @@
+ALTER TABLE 1_users ADD contacts text;