niche: add the templates and static files

Makefile

@@ -1,14 +0,0 @@
-all: version.py
-
-version.py: dummy
-	echo __version__ = \'$(shell git describe --always --dirty --long)\' > $@
-
-dummy:
-
-DB=$(shell cat niche.ini | grep ^db= | cut -d= -f2)
-
-import: $(wildcard mofi-*.sql.gz)
-	zcat $< \
-	| sed -r "s# CHARSET\=latin1# CHARSET\=utf8#g" \
-	| sed "s#ENGINE=MyISAM##" \
-	| mysql -u root -p $(DB)

NOTES.txt

@@ -18,3 +18,31 @@ mysql -u niche -p niche < schema.sql
 Sanitizing
 ----------
 bleach looks fine https://github.com/jsocol/bleach
+
+Performance
+-----------
+All with niche-dev running a local server.
+
+base: 1592 ms
+web.config.debug = False: 841 ms
+Pre-compiled templates: 848 ms
+Basic MySQL caching: ~366 ms
+
+cProfile doesn't do multiple threads.
+
+HTML templates: 366 ms
+Precompiled templates: 360 ms (no advantage)
+
+Baseline: 366 ms
+Basic caching: 228 ms
+Binary pickle: 221 ms
+
+Before launch
+-------------
+Error and access logging: DONE
+Check /static/ is served by nginx: DONE
+Check gzip effects: DONE
+Way of logging bytes transferred: 
+error.log has no SQL in it: DONE
+memcached in monit: DONE
+monit emails sending: DONE

niche/__main__.py

@@ -0,0 +1,4 @@
+from . import app
+
+if __name__ == "__main__":
+    app.main()

niche/app.py

@@ -2,7 +2,7 @@
 
 import datetime
 import hashlib
-import ConfigParser
+import configparser
 import passlib
 import markdown
 import re
@@ -12,16 +12,17 @@ import json
 import random
 import sys
 import pickle
-import urllib
+import urllib.request, urllib.parse, urllib.error
+import os.path
 
 import memcache
 import web
 import bleach
 from passlib.apps import custom_app_context as pwd_context
 
-import strings
+from . import strings
-import utils
+from . import utils
-import version
+from . import version
 
 web.config.debug = False
 
@@ -30,32 +31,56 @@ web.config.debug = False
 # pylint: disable=no-init
 
 urls = (
-    r'/?', 'index',
+    r'/?',
-    r'/links(/\d+)?(/\d+)?(/\d+)?', 'links',
+    'index',
-    r'/link/new', 'new_link',
+    r'/links(/\d+)?(/\d+)?(/\d+)?',
-    r'/link/(\d+)', 'link',
+    'links',
-    r'/link/(\d+)/hide', 'hide_link',
+    r'/link/new',
-    r'/link/(\d+)/close', 'close_link',
+    'new_link',
-    r'/link/(\d+)/new', 'new_comment',
+    r'/link/(\d+)',
-    r'/comment/(\d+)/delete', 'delete_comment',
+    'link',
-    r'/comment/(\d+)/like', 'like_comment',
+    r'/link/(\d+)/hide',
-    r'/user/([^/]+)', 'user',
+    'hide_link',
-    r'/user/([^/]+)/links', 'user_links',
+    r'/link/(\d+)/close',
-    r'/user/([^/]+)/comments', 'user_comments',
+    'close_link',
-    r'/user/([^/]+)/checkout', 'checkout',
+    r'/link/(\d+)/new',
-    r'/user/([^/]+)/password', 'password',
+    'new_comment',
-    r'/user/([^/]+)/edit', 'user_edit',
+    r'/comment/(\d+)/delete',
-    r'/login', 'login',
+    'delete_comment',
-    r'/logout', 'logout',
+    r'/comment/(\d+)/like',
-    r'/rss', 'rss',
+    'like_comment',
-    r'/debug/counters', 'debug_counters',
+    r'/user/([^/]+)',
-    r'/debug/diediedie', 'debug_die',
+    'user',
+    r'/user/([^/]+)/links',
+    'user_links',
+    r'/user/([^/]+)/comments',
+    'user_comments',
+    r'/user/([^/]+)/checkout',
+    'checkout',
+    r'/user/([^/]+)/password',
+    'password',
+    r'/user/([^/]+)/edit',
+    'user_edit',
+    r'/login',
+    'login',
+    r'/logout',
+    'logout',
+    r'/rss',
+    'rss',
+    r'/debug/counters',
+    'debug_counters',
+    r'/debug/diediedie',
+    'debug_die',
 
     # MonkeyFilter compatible URLs.
-    r'/link\.php/(\d+)', 'link',
+    r'/link\.php/(\d+)',
-    r'/user\.php/([^/]+)', 'user',
+    'link',
-    r'/rss.php', 'rss',
+    r'/user\.php/([^/]+)',
-    r'/rss.xml', 'rss',
+    'user',
+    r'/rss.php',
+    'rss',
+    r'/rss.xml',
+    'rss',
 )
 
 ALLOWED_TAGS = """
@@ -75,41 +100,51 @@ ALLOWED_ATTRIBUTES = {
 # Default configuration.
 DEFAULTS = [
     ('general', {
-            'dateformat': '%B %d, %Y',
+        'dateformat':
-            'base': '/',
+        '%B %d, %Y',
-            'extra_tags': '',
+        'base':
-            'limit': 20,
+        '/',
-            'server_type': 'dev',
+        'has_https':
-            'user_fields': ('realname email homepage gravatar_email '
+        False,
-                            'team location twitter facebook '
+        'extra_tags':
-                            'google_plus_ skype aim'),
+        '',
-            'history_days': 7,
+        'limit':
-            }),
+        20,
+        'server_type':
+        'dev',
+        'user_fields': ('realname email homepage gravatar_email '
+                        'team location twitter facebook '
+                        'google_plus_ skype aim'),
+        'history_days':
+        7,
+        'templates':
+        'templates',
+    }),
     ('groups', {
-            'admins': '',
+        'admins': '',
-            }),
+    }),
     ('db', {
-            'db': 'niche',
+        'db': 'niche',
-            'user': 'niche',
+        'user': 'niche',
-            'password': 'whatever',
+        'password': 'whatever',
-            }),
+    }),
     ('cache', {
-            'host': 'localhost:11211',
+        'host': 'localhost:11211',
-            'max_age': 15,
+        'max_age': 15,
-            }),
+    }),
     ('site', {
-            'name': 'Nichefilter',
+        'name': 'Nichefilter',
-            'subtitle': 'of no fixed subtitle',
+        'subtitle': 'of no fixed subtitle',
-            'contact': None,
+        'contact': None,
-            'license': None,
+        'license': None,
-            'secret': '',
+        'secret': '',
-            }),
+    }),
-    ]
+]
 
 counters = utils.Counters()
 
 
-class Config(ConfigParser.RawConfigParser):
+class Config(configparser.RawConfigParser):
     def set_defaults(self, defaults):
         for section, items in defaults:
             self.add_section(section)
@@ -128,6 +163,7 @@ def read_config():
     cfg.read('niche.ini')
     return cfg
 
+
 config = read_config()
 
 FEATURES = 'likes gravatar rss checkout'.split()
@@ -143,6 +179,7 @@ def get_features(config):
 
     return features
 
+
 features = get_features(config)
 
 
@@ -161,13 +198,15 @@ def get_string(id):
     id = re.sub(r'_$', '', id)
     return strings.__dict__[id]
 
+
 _ = get_string
 
-db = web.database(dbn='mysql',
+db = web.database(
-                  user=config.get('db', 'user'),
+    dbn='mysql',
-                  pw=config.get('db', 'password'),
+    user=config.get('db', 'user'),
-                  db=config.get('db', 'db'),
+    pw=config.get('db', 'password'),
-                  )
+    db=config.get('db', 'db'),
+)
 
 
 class DBCache:
@@ -183,12 +222,12 @@ class DBCache:
 
     def make_key(self, table, column, value, limit):
         return '/'.join((
-                self._prefix,
+            self._prefix,
-                table,
+            table,
-                column,
+            column,
-                hashlib.md5(str(value)).hexdigest(),
+            hashlib.md5(str(value).encode('utf-8')).hexdigest(),
-                str(limit)
+            str(limit),
-                ))
+        ))
 
     def select(self, table, column, value, limit=None):
         dirty = self.make_dirty_key(table)
@@ -199,8 +238,11 @@ class DBCache:
             counters.bump('select_cache_hit')
             return got[elem]
         else:
-            rows = list(self._db.select(table, where='%s = $value' % column,
+            rows = list(
-                                 vars={'value': value}, limit=limit))
+                self._db.select(table,
+                                where='%s = $value' % column,
+                                vars={'value': value},
+                                limit=limit))
             counters.bump('select_cache_miss')
             self._cache.set(elem, rows, time=self._max_age)
             return rows
@@ -211,7 +253,8 @@ class DBCache:
         dirty = self.make_dirty_key(table)
         self._db.update(table,
                         where='%s = $id' % column,
-                        vars={'id': id}, **kwargs)
+                        vars={'id': id},
+                        **kwargs)
         self._cache.set(dirty, 1)
 
     def insert(self, type, **kwargs):
@@ -221,6 +264,7 @@ class DBCache:
         self._cache.set(dirty, 1)
         return result
 
+
 cache = DBCache(db,
                 host=config.get('cache', 'host'),
                 max_age=config.get('cache', 'max_age'),
@@ -235,6 +279,7 @@ def require_feature(name):
 def now():
     return time.time()
 
+
 fallbacks = {
     'user': web.utils.Storage(username='anonymous'),
 }
@@ -295,9 +340,8 @@ class AutoMapper:
             field = name[:-6]
             query = ('SELECT COUNT(*) AS total FROM 1_%ss '
                      'WHERE %sID = $id') % (field, self._type)
-            results = db.query(
+            results = db.query(query,
-                query,
+                               vars={'id': getattr(self, '%sID' % self._type)})
-                vars={'id': getattr(self, '%sID' % self._type)})
             return results[0].total
 
         if name.endswith('s'):
@@ -374,9 +418,9 @@ def render_input(v, use_markdown=False):
         return v
 
     if use_markdown:
-        return bleach.clean(
+        return bleach.clean(markdown.markdown(v, output_format='html5'),
-            markdown.markdown(v, output_format='html5'),
+                            tags=tags,
-            tags=tags, attributes=attrs)
+                            attributes=attrs)
     else:
         v = bleach.clean(v, tags=tags, attributes=attrs)
         out = ''
@@ -421,13 +465,13 @@ class Model:
 
     def get_user_by_name(self, name):
         """Get a user by user name"""
-        name = urllib.unquote(name)
+        name = urllib.parse.unquote_plus(name)
         user = first('user', 'username', name)
         return first('user', 'userID', user.userID)
 
     def get_gravatar(self, email):
         """Get the gravatar hash for an email"""
-        return hashlib.md5(email.strip().lower()).hexdigest()
+        return hashlib.md5(email.strip().lower().encode('utf-8')).hexdigest()
 
     def get_message(self):
         """Get the message for the user, if any, and clear"""
@@ -453,6 +497,9 @@ class Model:
 
         return first_or_none('user', 'userID', id)
 
+    def is_active(self):
+        return session.get('userID', None) is not None
+
     def paginate(self, offset, total, per_page):
         # TODO(michaelh): really a helper, not part of the model.
         page = 1 + offset // per_page
@@ -462,13 +509,12 @@ class Model:
             if pages / step <= 6:
                 break
         if pages > 1:
-            indexes = set(
+            indexes = set([1, 2, pages, pages - 1, page] +
-                [1, 2, pages, pages-1, page]
+                          list(range(step, pages, step)))
-                + range(step, pages, step))
             if page > 1:
-                indexes.add(page-1)
+                indexes.add(page - 1)
             if page < pages:
-                indexes.add(page+1)
+                indexes.add(page + 1)
         else:
             indexes = []
         return page, sorted(indexes)
@@ -489,13 +535,15 @@ class Model:
             return '%d %ss' % (value, name)
 
     def get_new(self):
-        since = now() - 60*60*24*config.get('general', 'history_days')
+        since = now() - 60 * 60 * 24 * config.get('general', 'history_days')
-        comments = db.select(
+        comments = db.select('1_comments',
-            '1_comments',
+                             where='timestamp >= $since AND userID <> $user',
-            where='timestamp >= $since AND userID <> $user',
+                             order='timestamp ASC',
-            order='timestamp ASC', limit=50,
+                             limit=50,
-            vars={'since': since,
+                             vars={
-                  'user': session.get('userID', None)})
+                                 'since': since,
+                                 'user': session.get('userID', None)
+                             })
         # Pull out the unique links.
         ids = {}
         for comment in comments:
@@ -504,6 +552,7 @@ class Model:
         comments = sorted(ids.values(), key=lambda x: x.linkID)
         return [AutoMapper('comment', x) for x in comments]
 
+
 model = Model()
 
 render_globals = {
@@ -516,15 +565,15 @@ render_globals = {
 }
 
 render = web.template.render(
-    'templates/',
+    config.get('general', 'templates'),
     base='layout',
     globals=render_globals,
-    )
+)
 
 naked_render = web.template.render(
-    'templates/',
+    config.get('general', 'templates'),
     globals=render_globals,
-    )
+)
 
 app = web.application(urls, locals())
 
@@ -532,12 +581,10 @@ app = web.application(urls, locals())
 def get_csrf():
     token = session.get('csrf_token', None)
     if token is None:
-        token = hashlib.md5(''.join((
+        key = ''.join(
-                str(random.randrange(0, 2**20)),
+            (str(random.randrange(0, 2**20)), config.get('site', 'secret'),
-                config.get('site', 'secret'),
+             config.get('db', 'db'), config.get('db', 'user')))
-                config.get('db', 'db'),
+        token = hashlib.md5(key.encode('utf-8')).hexdigest()
-                config.get('db', 'user')))
-                            ).hexdigest()
         session.csrf_token = token
     return token
 
@@ -566,6 +613,7 @@ class CSRFInput(web.form.Hidden):
     def validate(self, value):
         return check_csrf(value)
 
+
 TEXT_SIZE = 80
 TEXT_MAX_LENGTH = 150
 
@@ -584,21 +632,21 @@ def tidy_form(form):
 def make_session():
     """Helper that makes the session object, even if in debug mode."""
     if web.config.get('_session') is None:
-        session = web.session.Session(app, web.session.DiskStore('sessions'),
+        session = web.session.Session(app,
-                                      initializer={'message': None}
+                                      web.session.DiskStore('sessions'),
-                                      )
+                                      initializer={'message': None})
         web.config._session = session
     else:
         session = web.config._session
 
     return session
 
+
 session = make_session()
 
 # Validate a password.  Pretty lax.
-password_validator = web.form.Validator(
+password_validator = web.form.Validator(_("Short password"),
-    _("Short password"),
+                                        lambda x: len(x) >= 3)
-    lambda x: len(x) >= 3)
 
 
 def url_validator(v):
@@ -610,7 +658,19 @@ def url_validator(v):
 
 def redirect(url):
     """Bounce to a different site absolute URL."""
-    raise web.seeother(url)
+    has_https = config.get('general', 'has_https')
+    if has_https and model.is_active():
+        base = config.get('general', 'base')
+        if base.endswith('/') and url.startswith('/'):
+            path = base[:-1] + url
+        else:
+            path = base + url
+
+        raise web.seeother('https://{host}{path}'.format(host=web.ctx.host,
+                                                         path=path),
+                           absolute=True)
+    else:
+        raise web.seeother(url)
 
 
 def authenticate(msg=_("Login required")):
@@ -659,22 +719,24 @@ def render_links(where=None, span=None, vars=None, date_range=None):
     limit = int(input.get('limit', config.get('general', 'limit')))
     limit = max(0, min(200, limit))
 
-    links = db.select('1_links', where=where, vars=vars,
+    links = db.select('1_links',
-                      limit=limit, offset=offset,
+                      where=where,
+                      vars=vars,
+                      limit=limit,
+                      offset=offset,
                       order="timestamp DESC")
 
     if where:
-        results = db.query(
+        results = db.query('SELECT COUNT(*) AS total FROM 1_links WHERE %s' %
-            'SELECT COUNT(*) AS total FROM 1_links WHERE %s' % where,
+                           where,
-            vars=vars)
+                           vars=vars)
     else:
         results = db.query("SELECT COUNT(*) AS total FROM 1_links")
 
     total = results[0].total
 
-    return render.links(map_all('link', links), span,
+    return render.links(map_all('link', links), span, web.ctx.path, offset,
-                        web.ctx.path, offset, limit,
+                        limit, total, date_range)
-                        total, date_range)
 
 
 class index:
@@ -733,18 +795,21 @@ class links:
         last = datetime.datetime.fromtimestamp(limits.last)
 
         date_range = web.utils.Storage(
-            years=range(first.year, last.year+1),
+            years=list(range(first.year, last.year + 1)),
             year=None if no_year else year,
             month=None if no_month else month,
             months=calendar.month_name,
-            )
+        )
 
         return render_links(
             where='timestamp >= $tstart and timestamp < $tend',
-            vars={'tstart': tstart, 'tend': tend},
+            vars={
+                'tstart': tstart,
+                'tend': tend
+            },
             span=span,
             date_range=date_range,
-            )
+        )
 
 
 class link:
@@ -758,22 +823,19 @@ class link:
 class new_link:
     form = web.form.Form(
         web.form.Textbox('title', web.form.notnull),
-        web.form.Textbox('url', web.form.Validator(
+        web.form.Textbox('url',
-                _("Not a URL"), url_validator)),
+                         web.form.Validator(_("Not a URL"), url_validator)),
         web.form.Textbox('url_description'),
         web.form.Textarea('description', rows=5, cols=80),
         web.form.Textarea('extended', rows=5, cols=80),
         web.form.Checkbox('use_markdown', value='use_markdown'),
         CSRFInput(),
         validators=[
-            web.form.Validator(
+            web.form.Validator(_("URLs need a description"),
-                _("URLs need a description"),
+                               lambda x: x.url_description if x.url else True),
-                lambda x: x.url_description if x.url else True),
+            web.form.Validator(_("Need a URL or description"),
-            web.form.Validator(
+                               lambda x: x.url or x.description),
-                _("Need a URL or description"),
+        ])
-                lambda x: x.url or x.description),
-            ]
-        )
     form = tidy_form(form)
 
     def authenticate(self):
@@ -799,12 +861,11 @@ class new_link:
         extended = render_input(form.d.extended, markdown)
 
         if 'preview' in web.input():
-            preview = web.utils.Storage(
+            preview = web.utils.Storage(title=form.d.title,
-                title=form.d.title,
+                                        URL=form.d.url,
-                URL=form.d.url,
+                                        URL_description=url_description,
-                URL_description=url_description,
+                                        description=description,
-                description=description,
+                                        extended=extended)
-                extended=extended)
 
             return render.new_link(form, preview)
 
@@ -815,8 +876,7 @@ class new_link:
                             URL=form.d.url,
                             URL_description=url_description,
                             description=description,
-                            extended=extended
+                            extended=extended)
-                            )
 
         model.inform(_("New post success"))
         redirect('/link/%d' % next)
@@ -853,7 +913,7 @@ class new_comment:
         web.form.Textarea('comment', web.form.notnull, rows=5, cols=80),
         web.form.Checkbox('use_markdown', value='use_markdown'),
         CSRFInput(),
-        )
+    )
     form = tidy_form(form)
 
     def check(self, id):
@@ -882,8 +942,7 @@ class new_comment:
                      linkID=link.linkID,
                      userID=user.userID,
                      timestamp=now(),
-                     content=comment
+                     content=comment)
-                     )
         model.inform(_("New comment success"))
         redirect('/link/%d' % link.linkID)
 
@@ -926,17 +985,18 @@ class user_links:
     def GET(self, id):
         counters.bump(self)
         target = model.get_user_by_name(id)
-        return render_links(where='userID=$id', vars={'id': target.userid})
+        return render_links(where='userID=$id', vars={'id': target.userID})
 
 
 class user_comments:
     def GET(self, id):
         counters.bump(self)
         userid = first('user', 'username', id).userID
-        comments = db.select(
+        comments = db.select('1_comments',
-            '1_comments', where='userID=$id',
+                             where='userID=$id',
-            order='timestamp DESC', vars={'id': userid},
+                             order='timestamp DESC',
-            limit=config.get('general', 'limit'))
+                             vars={'id': userid},
+                             limit=config.get('general', 'limit'))
         return render.user_comments(
             [AutoMapper('comment', x) for x in comments])
 
@@ -958,7 +1018,7 @@ class login:
         web.form.Textbox('username', web.form.notnull),
         web.form.Password('password', web.form.notnull),
         CSRFInput(),
-        )
+    )
     login = tidy_form(login)
 
     def GET(self):
@@ -996,18 +1056,16 @@ class logout:
 
 
 class password:
-    form = web.form.Form(
+    form = web.form.Form(web.form.Password('password', web.form.notnull),
-        web.form.Password('password', web.form.notnull),
+                         web.form.Password('new_password', web.form.notnull,
-        web.form.Password('new_password',
+                                           password_validator),
-                          web.form.notnull, password_validator),
+                         web.form.Password('again', web.form.notnull),
-        web.form.Password('again', web.form.notnull),
+                         CSRFInput(),
-        CSRFInput(),
+                         validators=[
-        validators=[
+                             web.form.Validator(
-            web.form.Validator(
+                                 _("Passwords don't match"),
-                _("Passwords don't match"),
+                                 lambda x: x.new_password == x.again)
-                lambda x: x.new_password == x.again)
+                         ])
-            ]
-        )
     form = tidy_form(form)
 
     def authenticate(self, name):
@@ -1035,7 +1093,8 @@ class password:
                 form.note = _('Bad password')
                 return render.password(form)
 
-        cache.update('user', target.userID,
+        cache.update('user',
+                     target.userID,
                      password=pwd_context.encrypt(form.d.new_password))
         model.inform(_("Password changed"))
         redirect('/user/%s' % name)
@@ -1051,8 +1110,8 @@ class user_edit:
             return value if value else user.get(name)
 
         fields = [web.form.Textbox(x, value=get(x), size=60) for x in names]
-        fields.append(web.form.Textarea('bio',
+        fields.append(
-                                        rows=5, cols=80, value=get('bio')))
+            web.form.Textarea('bio', rows=5, cols=80, value=get('bio')))
         fields.append(CSRFInput())
         return tidy_form(web.form.Form(*fields))
 
@@ -1126,5 +1185,6 @@ def main():
 
     app.run()
 
+
 if __name__ == "__main__":
     main()

niche/test_niche.py

@@ -0,0 +1,2 @@
+def test_loads():
+    import niche

niche/version.py

@@ -0,0 +1 @@
+__version__ = 'r1-5-gb99a317-dirty'

setup.py

@@ -0,0 +1,30 @@
+import glob
+
+from setuptools import find_packages, setup
+
+setup(
+    name='niche',
+    version_format='{tag}.dev{commitcount}+{gitsha}',
+    setup_requires=['setuptools-git-version'],
+    url='https://juju.nz/src/juju/niche',
+    author='Michael Hope',
+    author_email='michaelh@juju.nz',
+    description='A community web log for monkey niches.',
+    zip_safe=False,
+    packages=find_packages(),
+    data_files=[
+        ('lib/niche/templates', glob.glob('templates/*')),
+        ('lib/niche/static/themes/common',
+         glob.glob('static/themes/common/*')),
+        ('lib/niche/static/themes/able-wpcom',
+         glob.glob('static/themes/able-wpcom/*')),
+    ],
+    install_requires=[
+        'passlib>=1.7.4',
+        'bleach>=3.2.1',
+        'markdown>=3.1.1',
+        'python-memcached>=1.59',
+        'web.py>=0.62',
+        'pymysql>=0.9.3',
+    ],
+)

templates/layout.html

@@ -31,7 +31,11 @@ $ active = model.get_active()
               <li><a href="user/$active.username">$active.username</a>
               <li><a href="logout">Logout</a>
             $else:
-              <li><a href="login">Login</a>
+              <li>
+              $if config.get('general', 'has_https'):
+                <a href="https://$config.get('general', 'domain')${base}login">Login</a>
+              $else:
+                <li><a href="login">Login</a>
             <li><a href="link/new">New post</a>
             <li><a href="links">Archive</a>
             </ul>

templates/link.html

@@ -36,6 +36,7 @@ $for comment in comments:
     <li>
     <div class="comment" id="comment_$comment.commentID">
     $:comment.content
+    </b></i>
     <footer>
     $ user = comment.user
     posted <a href="$here#comment_$comment.commentID">$comment.ago()</a> ago

tests.mk

@@ -1,4 +1,5 @@
 GET = / \
+	/links /links/2010 /links/2010/12 \
 	/link/2 /link/100 \
 	/user/michaelh /user/tracicle \
 	/login \