Force unicode in templates, everywhere. (Introduces a new |u filter)

klaus.py

@@ -108,6 +108,26 @@ def guess_is_image(filename):
         return False
     return mime.startswith('image/')
 
+def force_unicode(s):
+    if isinstance(s, unicode):
+        return s
+    try:
+        return s.decode('utf-8')
+    except UnicodeDecodeError as exc:
+        pass
+    try:
+        return s.decode('iso-8859-1')
+    except UnicodeDecodeError:
+        pass
+    try:
+        import chardet
+        encoding = chardet.detect(s)['encoding']
+        if encoding is not None:
+            return s.decode(encoding)
+    except (ImportError, UnicodeDecodeError):
+        raise exc
+
+app.jinja_env.filters['u'] = force_unicode
 app.jinja_env.filters['timesince'] = timesince
 app.jinja_env.filters['shorten_id'] = lambda id: id[:7] if len(id) in {20, 40} else id
 app.jinja_env.filters['shorten_message'] = lambda msg: msg.split('\n')[0]
@@ -244,6 +264,7 @@ class BlobView(BaseBlobView, TreeView):
         super(BlobView, self).view()
         self['tree'] = self.listdir(self['directory'])
         self['raw_url'] = self.build_url('raw_blob')
+        self['too_large'] = sum(map(len, self['blob'].chunked)) > 100*1024
 
     def get_parent_directory(self):
         return self['directory']

repo.py

@@ -90,7 +90,7 @@ class RepoWrapper(dulwich.repo.Repo):
         return ((entry.path, entry.in_path(root)) for entry in tree.iteritems())
 
     def commit_diff(self, commit):
-        from klaus import guess_is_binary
+        from klaus import guess_is_binary, force_unicode
 
         if commit.parents:
             parent_tree = self[commit.parents[0]].tree
@@ -118,7 +118,7 @@ class RepoWrapper(dulwich.repo.Repo):
             dulwich.patch.write_object_diff(stringio, self.object_store,
                                             (oldpath, oldmode, oldsha),
                                             (newpath, newmode, newsha))
-            files = prepare_udiff(stringio.getvalue().decode('utf-8'),
+            files = prepare_udiff(force_unicode(stringio.getvalue()),
                                   want_header=False)
             if not files:
                 # the diff module doesn't handle deletions/additions

templates/base.html

@@ -9,9 +9,9 @@
   {% for name, subpath in subpaths %}
     <span>
       {% if loop.last %}
-      <a href="">{{ name }}</a>
+      <a href="">{{ name|u }}</a>
       {% else %}
-      <a href="{{ build_url('view_tree', repo=repo.name, commit_id=commit_id, path=subpath) }}">{{ name }}</a>
+      <a href="{{ build_url('view_tree', repo=repo.name, commit_id=commit_id, path=subpath) }}">{{ name|u }}</a>
       {% endif %}
     </span>
   {% endfor %}

templates/history.inc.html

@@ -11,10 +11,10 @@
     <li>
       <a class=commit href="{{ build_url('view_commit', repo=repo.name, commit_id=commit.id) }}">
         <span class=line1>
-          <span>{{ commit.message.decode('utf-8')|shorten_message }}</span>
+          <span>{{ commit.message|u|shorten_message }}</span>
         </span>
         <span class=line2>
-          <span>{{ commit.author.decode('utf-8') }}</span>
+          <span>{{ commit.author|u }}</span>
           <span>{{ commit.commit_time|timesince }} ago</span>
         </span>
         <span class=clearfloat></span>

templates/tree.inc.html

 <div class=tree>
   <h2>Tree @<a href="{{ build_url('view_commit', repo=repo.name, commit_id=commit_id) }}">{{ commit_id|shorten_id }}</a></h2>
   <ul>
-    {% for name, fullpath in tree.dirs %}
-    <li><a href="{{ build_url('view_tree', repo=repo.name, commit_id=commit_id, path=fullpath) }}" class=dir>{{ name }}</a></li>
+    {% for _, name, fullpath in tree.dirs %}
+    <li><a href="{{ build_url('view_tree', repo=repo.name, commit_id=commit_id, path=fullpath) }}" class=dir>{{ name|u }}</a></li>
     {% endfor %}
-    {% for name, fullpath in tree.files %}
-    <li><a href="{{ build_url('view_blob', repo=repo.name, commit_id=commit_id, path=fullpath) }}">{{ name }}</a></li>
+    {% for _, name, fullpath in tree.files %}
+    <li><a href="{{ build_url('view_blob', repo=repo.name, commit_id=commit_id, path=fullpath) }}">{{ name|u }}</a></li>
     {% endfor %}
   </ul>
 </div>

templates/view_blob.html

@@ -5,7 +5,7 @@
 
 <div class=blobview>
   <h2>
-    {{ filename }}
+    {{ filename|u }}
     <span>
       @<a href="{{ build_url('view_commit', repo=repo.name, commit_id=commit_id) }}">{{ commit_id|shorten_id }}</a>
       (<a href="{{ raw_url }}">raw</a>)
@@ -18,7 +18,11 @@
       <div class=binary-warning>(Binary data not shown)</div>
     {% endif %}
   {% else %}
-    {{ blob.data.decode('utf-8')|pygmentize(filename=filename) }}
+    {% if too_large %}
+      <div class=too-large-warning>(Large file not shown)</div>
+    {% else %}
+      {{ blob.data|u|pygmentize(filename=filename) }}
+    {% endif %}
   {% endif %}
 </div>
 

templates/view_commit.html

@@ -2,7 +2,7 @@
 {% extends 'base.html' %}
 
 {% block h1 %}
-  Commit "{{commit.message.decode('utf-8')|shorten_message}}" to
+  Commit "{{commit.message|u|shorten_message}}" to
   <a href="{{ build_url('view_repo', repo=repo.name) }}">{{ repo.name }}</a>
 {% endblock %}
 
@@ -11,10 +11,10 @@
 <div class=full-commit>
   <div class=commit>
     <span class=line1>
-      <span>{{ commit.message.decode('utf-8') }}</span>
+      <span>{{ commit.message|u }}</span>
     </span>
     <span class=line2>
-      <span>{{ commit.author.decode('utf-8') }}</span>
+      <span>{{ commit.author|u }}</span>
       <span>{{ commit.commit_time|timesince }} ago</span>
     </span>
     <span class=clearfloat></span>
@@ -27,10 +27,10 @@
           {{ file.old_filename }} →
         {% endif %}#}
           {% if file.new_filename == 'dev/null' %}
-            <del>{{ file.old_filename }}</del>
+            <del>{{ file.old_filename|u }}</del>
           {% else %}
             <a href="{{ build_url('view_blob', repo=repo.name, commit_id=commit_id, path=file.new_filename) }}">
-              {{ file.new_filename }}
+              {{ file.new_filename|u }}
             </a>
           {% endif %}
       </div>
@@ -40,7 +40,7 @@
             <tr>
               <td class=linenos>{{ line.old_lineno }}</td>
               <td class=linenos>{{ line.new_lineno }}</td>
-              <td class={{line.action}}>{{ line.line }}</td>
+              <td class={{line.action}}>{{ line.line|u }}</td>
             </tr>
           {%- endfor %}
           {% if not loop.last %}